From b92cf5aa392a6b2ffff0cfd0ed08f76ee8cbfc54 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Wed, 12 Apr 2023 13:18:59 +0800 Subject: [PATCH 001/494] Release 3.0.0 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 132 files changed, 134 insertions(+), 134 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index cc7e7952b69f0..bc7d77ad4f0b4 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 44f0ada4630d7..cef09439b1e0f 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index bd5d64bd84191..732ade6a10f3e 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 9f8ace79b77c3..a93131a7ce43b 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index b22abf286f693..704a221155854 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,12 +31,12 @@ org.apache.pulsar buildtools - 3.0.0-SNAPSHOT + 3.0.0 jar Pulsar Build Tools - ${maven.build.timestamp} + 2023-04-12T05:18:48Z 1.8 1.8 3.0.0-M3 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 99105bef950d5..72b941f06ab99 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 1e86758ed5a6a..035cd23e62579 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 9782f269284bf..67846dae80084 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 6b225bfc00c0b..a4e20e62a2fc6 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index b38baee4257ba..4ca155f395ae0 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/docker/pom.xml b/docker/pom.xml index 4d3b05fe33a76..110a3876acaf8 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index c63ff6d656957..b4186c8c1a9a7 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 647f68bf1672c..d83755846b460 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index d4138ea041317..e673ec0404f0a 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 2a7b9c576d318..d3c5b9c0f4082 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pom.xml b/pom.xml index 094a35b5ed13a..8daa95e3642b8 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -92,7 +92,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - ${maven.build.timestamp} + 2023-04-12T05:18:48Z true diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index 419346c7adb90..ac9e0a59abc9f 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index 9ad0363775a13..6eaa665394504 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-broker-auth-oidc diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 8cde4a2fc4552..16d518eae76ce 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 3e024d14b92a0..7ce9e22a81d3a 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-broker-common diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 7442d95e467e6..31b335e1aea68 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index f039c23f92cb3..11fea04a6d364 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index a0a254317bfc8..3ad0cbb4ee287 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index bc5e003b108d7..ce3055d31e2de 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 69c423e86e246..6c09f0f063801 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index cf669b5869afb..ae92f1c7a0d85 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 9436f9c1ce245..597db6553ed39 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 73621954e1fb8..20be552aed125 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index 242cedc594e4e..4f72e08e6d25e 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 3cdbdb48a2935..b18157ba85bac 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index c1143bf69331e..dbcabe72b143e 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 6292d99003dd6..60091f3b1ca63 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index bf3d8a802dff2..7a5e177cc15f4 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-tools-api/pom.xml b/pulsar-client-tools-api/pom.xml index d6420a69e3a2a..48e8572bd6a5d 100644 --- a/pulsar-client-tools-api/pom.xml +++ b/pulsar-client-tools-api/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index f1f9180e2d550..73692050a8b77 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -22,7 +22,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. 4.0.0 diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 9edb3c1129ed5..a820292007409 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 3a0b03d2c616e..ee74265b2c753 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 9d0dd08eecf10..e43d8415dad98 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 643b03a59c0f7..8e7c8d9ef9a5b 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 5ce8f74d6ebad..3c38bca2c555c 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 0f6e33bc05de5..34a811d04e130 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 15b3eff0b6339..fe7a199bd7714 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions-instance diff --git a/pulsar-functions/java-examples-builtin/pom.xml b/pulsar-functions/java-examples-builtin/pom.xml index fe5449372f199..a1d095813f7bd 100644 --- a/pulsar-functions/java-examples-builtin/pom.xml +++ b/pulsar-functions/java-examples-builtin/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions-api-examples-builtin diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 14e1e17d282ca..f3b0d18ae9269 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 2626732457b00..3eab1a772b6d9 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index 6dcac5c4ac2be..b9bd58d11d587 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 3176aba9c9188..76f602622ef03 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index f188a147b2cee..36034de2f1d42 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index bf41c9b400be5..a95c5dbfba82f 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 5558ced4c0749..1789c85f62be5 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 08d4e9bf63ab8..6bf93d5a03cb7 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index a0dd90d5577af..305cd523e2210 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 1201c5aea5d22..803ddfbcb05cc 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 9f058741cafbd..96f0822f796fb 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-aerospike diff --git a/pulsar-io/alluxio/pom.xml b/pulsar-io/alluxio/pom.xml index b0f6a51b79cf4..d299682867e4f 100644 --- a/pulsar-io/alluxio/pom.xml +++ b/pulsar-io/alluxio/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 2c5ba9450d258..c7a3985bec99e 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index 621dd226f0eef..cdd61b0ed5d4c 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index 6b07bf4ecc8db..a43c836cc6324 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index 76eb484c8a56f..afcb6bd31406a 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 4d9296ea2b35c..82c22a041ae98 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index b975d9fef07f6..d006c118c6b25 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index 0d253bbbfc5ca..c6bfd3867253e 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 2ee0120c98836..47d7233555fc5 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 4588019a9aa97..f90a590d5aead 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index b672d64e6147f..34e2482928deb 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 1151940d770f4..dceb01a7d9652 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 47bb9506cb851..ab1ed50f04c93 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index ee158d6fb8414..f36dfcc49c7d7 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 050da1af45330..9d6992ff043e5 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 0bab38c71ef10..d2823fca2c845 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 305c7f1473077..559f2d01337db 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index 4233601180caa..b8a79f21d0b6d 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index f9f87e5bd02b2..8d87a9ea68555 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index b6e5a13679663..46f2fd19969fb 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index a27540683a3d0..5df8971e11790 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 33396394de4a4..78a19f3e43ed1 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 1f43b31a0aa6a..04ee350408dcd 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 3fbc030184c5c..dafcc53b5106d 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/http/pom.xml b/pulsar-io/http/pom.xml index d86201a67b30f..e0d07c3ce53d7 100644 --- a/pulsar-io/http/pom.xml +++ b/pulsar-io/http/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-http diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 91571cf66e803..914cc2d106b00 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 89ac29d72653a..56df558bfc4af 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index f9dd1e0e96346..cda23a9114431 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 7e96a2901456f..7e9c7b8461b5a 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-io/jdbc/openmldb/pom.xml b/pulsar-io/jdbc/openmldb/pom.xml index 1b2bab67d04ab..f1f27176e09b1 100644 --- a/pulsar-io/jdbc/openmldb/pom.xml +++ b/pulsar-io/jdbc/openmldb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 3f841ab183399..3590c1fdb1ebf 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -33,7 +33,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 3ae7eaf93bb7c..a15b0ee2824d2 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 6c688af0ae15e..29543c4d1c7ed 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index bd742f0f20332..c870da9d4236a 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 12ebda087eb3d..f4f8c9cb3dbe9 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 9e322961889ed..548b247f37718 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index b957a770dfcb0..c28fafa2353cc 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 63cdd397f2548..51d78863023a4 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index b6959199053e3..05b38952a29e7 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index 8f7307843627f..a574999a255e4 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 2e62509c08651..0b6b37cb7eec4 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 8fc0535c16089..72e5d5f9e59c9 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 3f04f6645697f..69fb283b28d97 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 4ef67a31ec33c..02698a27937b1 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 901f8639a4326..d2702ce1c8c6e 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0-SNAPSHOT + 3.0.0 pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 4096a1ee0f994..8559b38d02004 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index 274e5abb4abec..1022fa3fbb5e0 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index a54d2b1d8ad54..65f70c0c1d038 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-package-management/filesystem-storage/pom.xml b/pulsar-package-management/filesystem-storage/pom.xml index 94e075fea1973..e2f824d440b00 100644 --- a/pulsar-package-management/filesystem-storage/pom.xml +++ b/pulsar-package-management/filesystem-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index f05798ecb4bb3..926117693c992 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index b9a68cba2c5e1..555228d618cb9 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 1f8972bf22219..77fbedeee889a 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 48a1bf90662f6..940d38cc9b0b8 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 3.0.0-SNAPSHOT + 3.0.0 pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index 11d2cbdfa9925..510d9fcd379a5 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 3.0.0-SNAPSHOT + 3.0.0 pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index e61580f1fcf15..f34288d1fcb78 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 3.0.0-SNAPSHOT + 3.0.0 pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index ceb64ecbadd90..085931a896af1 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index 2b1cf90a2cda7..165616ad70f2d 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 3.0.0-SNAPSHOT + 3.0.0 pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index 25390007b7056..707698d3754ef 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 3.0.0-SNAPSHOT + 3.0.0 pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 133f1fe826e2b..b2808e6e8fb16 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 3f8f91d1416ba..d3425d4f8e690 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 8c058306fc278..b5c40b06d3405 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 1cdcd7906079f..3e2d47c59e1a0 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 3.0.0-SNAPSHOT + 3.0.0 testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 93adaffe83459..be09b3244c985 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0-SNAPSHOT + 3.0.0 bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 41da961f3bb8e..d3fa3bbb779d1 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0-SNAPSHOT + 3.0.0 bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index f1e75c41ae6d3..b970a5b2320c9 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index db685d0394253..526620e8995f8 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 5154c4e5599e5..8deaed18f9e95 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 java-test-image diff --git a/tests/docker-images/java-test-plugins/pom.xml b/tests/docker-images/java-test-plugins/pom.xml index b214c84bb6c36..045cec02207c9 100644 --- a/tests/docker-images/java-test-plugins/pom.xml +++ b/tests/docker-images/java-test-plugins/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 java-test-plugins diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 8474f820c9566..82e966230241c 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.0.0-SNAPSHOT + 3.0.0 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index ade69e0778b04..50ed707cd0b07 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0-SNAPSHOT + 3.0.0 docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index d9b817ca2e2b5..5b6f825f7f975 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0-SNAPSHOT + 3.0.0 integration diff --git a/tests/pom.xml b/tests/pom.xml index 8a4aec9504ff3..b5ef1a68ad5f7 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 0e7bed47db060..2012cca57dce2 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0-SNAPSHOT + 3.0.0 pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 2b65f49a360ae..d2b8df056be58 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0-SNAPSHOT + 3.0.0 pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 6efd88f4ac762..44e307bc1513c 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0-SNAPSHOT + 3.0.0 pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 745da1a95c70d..9a19f4eeaaecd 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 56ec203b47690..d95444e0e8aa1 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 3.0.0-SNAPSHOT + 3.0.0 .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index 6057e082fca01..b6359869e31c3 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0-SNAPSHOT + 3.0.0 .. From c8d2bba651b45b9345ba3f3c4e5bcfe2468a2206 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Thu, 13 Apr 2023 10:56:22 -0700 Subject: [PATCH 002/494] [improve][broker] PIP-192 Fix getLastMessageId for compressed payload(And add compression and maxBatchSize for the load balance system topic) (#20087) --- .../channel/ServiceUnitStateChannelImpl.java | 4 ++ .../pulsar/broker/service/BrokerService.java | 6 +- .../pulsar/broker/service/ServerCnx.java | 11 ++++ .../impl/RawBatchMessageContainerImpl.java | 12 ++-- .../StrategicTwoPhaseCompactor.java | 17 ++++- .../RawBatchMessageContainerImplTest.java | 63 +++++++++++++------ .../GetLastMessageIdCompactedTest.java | 43 +++++++++++++ .../ServiceUnitStateCompactionTest.java | 24 +++++-- 8 files changed, 145 insertions(+), 35 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index bd62c53be6083..68c6440e68e4b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -81,6 +81,7 @@ import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException; +import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; @@ -106,6 +107,8 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { TopicDomain.persistent.value(), SYSTEM_NAMESPACE, "loadbalancer-service-unit-state").toString(); + + public static final CompressionType MSG_COMPRESSION_TYPE = CompressionType.ZSTD; private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec public static final long VERSION_ID_INIT = 1; // initial versionId private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60; @@ -285,6 +288,7 @@ public synchronized void start() throws PulsarServerException { producer = pulsar.getClient().newProducer(schema) .enableBatching(true) + .compressionType(MSG_COMPRESSION_TYPE) .maxPendingMessages(MAX_OUTSTANDING_PUB_MESSAGES) .blockIfQueueFull(true) .topic(TOPIC) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index bb08734298f04..33e5500d623d2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2144,9 +2144,9 @@ public CompletableFuture checkTopicNsOwnership(final String topic) { if (ownedByThisInstance) { return CompletableFuture.completedFuture(null); } else { - String msg = String.format("Namespace bundle for topic (%s) not served by this instance. " - + "Please redo the lookup. Request is denied: namespace=%s", topic, - topicName.getNamespace()); + String msg = String.format("Namespace bundle for topic (%s) not served by this instance:%s. " + + "Please redo the lookup. Request is denied: namespace=%s", + topic, pulsar.getLookupServiceAddress(), topicName.getNamespace()); log.warn(msg); return FutureUtil.failedFuture(new ServiceUnitNotReadyException(msg)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 01274ab4460cb..888668e15b167 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -129,6 +129,7 @@ import org.apache.pulsar.common.api.proto.CommandUnsubscribe; import org.apache.pulsar.common.api.proto.CommandWatchTopicList; import org.apache.pulsar.common.api.proto.CommandWatchTopicListClose; +import org.apache.pulsar.common.api.proto.CompressionType; import org.apache.pulsar.common.api.proto.FeatureFlags; import org.apache.pulsar.common.api.proto.KeySharedMeta; import org.apache.pulsar.common.api.proto.KeySharedMode; @@ -141,6 +142,8 @@ import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.api.proto.SingleMessageMetadata; import org.apache.pulsar.common.api.proto.TxnAction; +import org.apache.pulsar.common.compression.CompressionCodec; +import org.apache.pulsar.common.compression.CompressionCodecProvider; import org.apache.pulsar.common.intercept.InterceptException; import org.apache.pulsar.common.naming.Metadata; import org.apache.pulsar.common.naming.NamespaceName; @@ -2167,6 +2170,14 @@ private int calculateTheLastBatchIndexInBatch(MessageMetadata metadata, ByteBuf if (batchSize <= 1){ return -1; } + if (metadata.hasCompression()) { + var tmp = payload; + CompressionType compressionType = metadata.getCompression(); + CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); + int uncompressedSize = metadata.getUncompressedSize(); + payload = codec.decode(payload, uncompressedSize); + tmp.release(); + } SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata(); int lastBatchIndexInBatch = -1; for (int i = 0; i < batchSize; i++){ diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java index cf6b213155cd7..7e1c2cd5e3fe3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java @@ -47,13 +47,13 @@ public class RawBatchMessageContainerImpl extends BatchMessageContainerImpl { MessageCrypto msgCrypto; Set encryptionKeys; CryptoKeyReader cryptoKeyReader; - public RawBatchMessageContainerImpl(int maxNumMessagesInBatch) { + + public RawBatchMessageContainerImpl(int maxNumMessagesInBatch, int maxBytesInBatch) { super(); - compressionType = CompressionType.NONE; - compressor = new CompressionCodecNone(); - if (maxNumMessagesInBatch > 0) { - this.maxNumMessagesInBatch = maxNumMessagesInBatch; - } + this.compressionType = CompressionType.NONE; + this.compressor = new CompressionCodecNone(); + this.maxNumMessagesInBatch = maxNumMessagesInBatch; + this.maxBytesInBatch = maxBytesInBatch; } private ByteBuf encrypt(ByteBuf compressedPayload) { if (msgCrypto == null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java index 37b03e275d6bf..557d4a6580161 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.compaction; +import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import java.time.Duration; import java.util.Iterator; @@ -62,17 +63,29 @@ public class StrategicTwoPhaseCompactor extends TwoPhaseCompactor { private static final Logger log = LoggerFactory.getLogger(StrategicTwoPhaseCompactor.class); private static final int MAX_OUTSTANDING = 500; + private static final int MAX_NUM_MESSAGES_IN_BATCH = 1000; + private static final int MAX_BYTES_IN_BATCH = 128 * 1024; private static final int MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS = 20 * 1000; private final Duration phaseOneLoopReadTimeout; private final RawBatchMessageContainerImpl batchMessageContainer; + @VisibleForTesting public StrategicTwoPhaseCompactor(ServiceConfiguration conf, PulsarClient pulsar, BookKeeper bk, ScheduledExecutorService scheduler, int maxNumMessagesInBatch) { + this(conf, pulsar, bk, scheduler, maxNumMessagesInBatch, MAX_BYTES_IN_BATCH); + } + + private StrategicTwoPhaseCompactor(ServiceConfiguration conf, + PulsarClient pulsar, + BookKeeper bk, + ScheduledExecutorService scheduler, + int maxNumMessagesInBatch, + int maxBytesInBatch) { super(conf, pulsar, bk, scheduler); - batchMessageContainer = new RawBatchMessageContainerImpl(maxNumMessagesInBatch); + batchMessageContainer = new RawBatchMessageContainerImpl(maxNumMessagesInBatch, maxBytesInBatch); phaseOneLoopReadTimeout = Duration.ofSeconds(conf.getBrokerServiceCompactionPhaseOneLoopTimeInSeconds()); } @@ -80,7 +93,7 @@ public StrategicTwoPhaseCompactor(ServiceConfiguration conf, PulsarClient pulsar, BookKeeper bk, ScheduledExecutorService scheduler) { - this(conf, pulsar, bk, scheduler, -1); + this(conf, pulsar, bk, scheduler, MAX_NUM_MESSAGES_IN_BATCH, MAX_BYTES_IN_BATCH); } public CompletableFuture compact(String topic) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java index 9fa834a166cda..9b8b1e5efb99c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java @@ -19,8 +19,10 @@ package org.apache.pulsar.client.impl; -import static org.apache.pulsar.common.api.proto.CompressionType.LZ4; import static org.apache.pulsar.common.api.proto.CompressionType.NONE; +import static org.apache.pulsar.common.api.proto.CompressionType.ZSTD; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertTrue; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.io.IOException; @@ -45,6 +47,7 @@ import org.apache.pulsar.compaction.CompactionTest; import org.testng.Assert; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class RawBatchMessageContainerImplTest { @@ -53,9 +56,11 @@ public class RawBatchMessageContainerImplTest { CryptoKeyReader cryptoKeyReader; Map encryptKeys; + int maxBytesInBatch = 5 * 1024 * 1024; + public void setEncryptionAndCompression(boolean encrypt, boolean compress) { if (compress) { - compressionType = LZ4; + compressionType = ZSTD; } else { compressionType = NONE; } @@ -100,14 +105,24 @@ public MessageImpl createMessage(String topic, String value, int entryId) { @BeforeMethod public void setup() throws Exception { - setEncryptionAndCompression(false, false); + setEncryptionAndCompression(false, true); } - @Test - public void testToByteBuf() throws IOException { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2); + @DataProvider(name = "testBatchLimitByMessageCount") + public static Object[][] testBatchLimitByMessageCount() { + return new Object[][] {{true}, {false}}; + } + + @Test(timeOut = 20000, dataProvider = "testBatchLimitByMessageCount") + public void testToByteBufWithBatchLimit(boolean testBatchLimitByMessageCount) throws IOException { + RawBatchMessageContainerImpl container = testBatchLimitByMessageCount ? + new RawBatchMessageContainerImpl(2, Integer.MAX_VALUE) : + new RawBatchMessageContainerImpl(Integer.MAX_VALUE, 5); + String topic = "my-topic"; - container.add(createMessage(topic, "hi-1", 0), null); - container.add(createMessage(topic, "hi-2", 1), null); + var full1 = container.add(createMessage(topic, "hi-1", 0), null); + var full2 = container.add(createMessage(topic, "hi-2", 1), null); + assertFalse(full1); + assertTrue(full2); ByteBuf buf = container.toByteBuf(); @@ -126,18 +141,23 @@ public void testToByteBuf() throws IOException { MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder(); Assert.assertEquals(metadata.getNumMessagesInBatch(), 2); Assert.assertEquals(metadata.getHighestSequenceId(), 1); - Assert.assertEquals(metadata.getCompression(), NONE); + Assert.assertEquals(metadata.getCompression(), ZSTD); + + CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); + ByteBuf payload = codec.decode(metadataAndPayload, metadata.getUncompressedSize()); SingleMessageMetadata messageMetadata = new SingleMessageMetadata(); + messageMetadata.setCompactedOut(true); ByteBuf payload1 = Commands.deSerializeSingleMessageInBatch( - singleMessageMetadataAndPayload.getPayload(), messageMetadata, 0, 2); + payload, messageMetadata, 0, 2); ByteBuf payload2 = Commands.deSerializeSingleMessageInBatch( - singleMessageMetadataAndPayload.getPayload(), messageMetadata, 1, 2); + payload, messageMetadata, 1, 2); Assert.assertEquals(payload1.toString(Charset.defaultCharset()), "hi-1"); Assert.assertEquals(payload2.toString(Charset.defaultCharset()), "hi-2"); payload1.release(); payload2.release(); + payload.release(); singleMessageMetadataAndPayload.release(); metadataAndPayload.release(); buf.release(); @@ -147,7 +167,7 @@ public void testToByteBuf() throws IOException { public void testToByteBufWithCompressionAndEncryption() throws IOException { setEncryptionAndCompression(true, true); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch); container.setCryptoKeyReader(cryptoKeyReader); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); @@ -169,7 +189,7 @@ public void testToByteBufWithCompressionAndEncryption() throws IOException { MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder(); Assert.assertEquals(metadata.getNumMessagesInBatch(), 2); Assert.assertEquals(metadata.getHighestSequenceId(), 1); - Assert.assertEquals(metadata.getCompression(), compressionType); + Assert.assertEquals(metadata.getCompression(), ZSTD); ByteBuf payload = singleMessageMetadataAndPayload.getPayload(); int maxDecryptedSize = msgCrypto.getMaxOutputSize(payload.readableBytes()); @@ -197,7 +217,7 @@ public void testToByteBufWithCompressionAndEncryption() throws IOException { @Test public void testToByteBufWithSingleMessage() throws IOException { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); ByteBuf buf = container.toByteBuf(); @@ -218,9 +238,12 @@ public void testToByteBufWithSingleMessage() throws IOException { MessageMetadata metadata = singleMessageMetadataAndPayload.getMessageBuilder(); Assert.assertEquals(metadata.getNumMessagesInBatch(), 1); Assert.assertEquals(metadata.getHighestSequenceId(), 0); - Assert.assertEquals(metadata.getCompression(), NONE); + Assert.assertEquals(metadata.getCompression(), ZSTD); + + CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); + ByteBuf payload = codec.decode(metadataAndPayload, metadata.getUncompressedSize()); - Assert.assertEquals(singleMessageMetadataAndPayload.getPayload().toString(Charset.defaultCharset()), "hi-1"); + Assert.assertEquals(payload.toString(Charset.defaultCharset()), "hi-1"); singleMessageMetadataAndPayload.release(); metadataAndPayload.release(); buf.release(); @@ -228,7 +251,7 @@ public void testToByteBufWithSingleMessage() throws IOException { @Test public void testMaxNumMessagesInBatch() { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); String topic = "my-topic"; boolean isFull = container.add(createMessage(topic, "hi", 0), null); @@ -238,14 +261,14 @@ public void testMaxNumMessagesInBatch() { @Test(expectedExceptions = UnsupportedOperationException.class) public void testCreateOpSendMsg() { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); container.createOpSendMsg(); } @Test public void testToByteBufWithEncryptionWithoutCryptoKeyReader() { setEncryptionAndCompression(true, false); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); Assert.assertEquals(container.getNumMessagesInBatch(), 1); @@ -263,7 +286,7 @@ public void testToByteBufWithEncryptionWithoutCryptoKeyReader() { @Test public void testToByteBufWithEncryptionWithInvalidEncryptKeys() { setEncryptionAndCompression(true, false); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); container.setCryptoKeyReader(cryptoKeyReader); encryptKeys = new HashMap<>(); encryptKeys.put(null, null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java index 0be9fa407542f..317b1a227e585 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/GetLastMessageIdCompactedTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; @@ -263,6 +264,48 @@ public void testGetLastMessageIdAfterCompaction(boolean enabledBatch) throws Exc admin.topics().delete(topicName, false); } + @Test(dataProvider = "enabledBatch") + public void testGetLastMessageIdAfterCompactionWithCompression(boolean enabledBatch) throws Exception { + String topicName = "persistent://public/default/" + BrokerTestUtil.newUniqueName("tp"); + String subName = "sub"; + Consumer consumer = createConsumer(topicName, subName); + var producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .batchingMaxPublishDelay(3, TimeUnit.HOURS) + .batchingMaxBytes(Integer.MAX_VALUE) + .compressionType(CompressionType.ZSTD) + .enableBatching(enabledBatch).create(); + + List> sendFutures = new ArrayList<>(); + sendFutures.add(producer.newMessage().key("k0").value("v0").sendAsync()); + sendFutures.add(producer.newMessage().key("k0").value("v1").sendAsync()); + sendFutures.add(producer.newMessage().key("k0").value("v2").sendAsync()); + producer.flush(); + sendFutures.add(producer.newMessage().key("k1").value("v0").sendAsync()); + sendFutures.add(producer.newMessage().key("k1").value("v1").sendAsync()); + sendFutures.add(producer.newMessage().key("k1").value("v2").sendAsync()); + producer.flush(); + FutureUtil.waitForAll(sendFutures).join(); + + triggerCompactionAndWait(topicName); + + MessageIdImpl lastMessageIdByTopic = getLastMessageIdByTopic(topicName); + MessageIdImpl messageId = (MessageIdImpl) consumer.getLastMessageId(); + assertEquals(messageId.getLedgerId(), lastMessageIdByTopic.getLedgerId()); + assertEquals(messageId.getEntryId(), lastMessageIdByTopic.getEntryId()); + if (enabledBatch){ + BatchMessageIdImpl lastBatchMessageIdByTopic = (BatchMessageIdImpl) lastMessageIdByTopic; + BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId; + assertEquals(batchMessageId.getBatchSize(), lastBatchMessageIdByTopic.getBatchSize()); + assertEquals(batchMessageId.getBatchIndex(), lastBatchMessageIdByTopic.getBatchIndex()); + } + + // cleanup. + consumer.close(); + producer.close(); + admin.topics().delete(topicName, false); + } + @Test(dataProvider = "enabledBatch") public void testGetLastMessageIdAfterCompactionEndWithNullMsg(boolean enabledBatch) throws Exception { String topicName = "persistent://public/default/" + BrokerTestUtil.newUniqueName("tp"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 02812898dc49a..1a69a86f7c6b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -25,6 +25,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Splitting; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -185,6 +186,7 @@ TestData generateTestData() throws PulsarAdminException, PulsarClientException { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); @@ -352,12 +354,13 @@ public void testCompactionWithTableview() throws Exception { compactor.compact(topic, strategy).get(); // consumer with readCompacted enabled only get compacted entries - var tableview = pulsar.getClient().newTableViewBuilder(schema) + var tableview = pulsar.getClient().newTableView(schema) .topic(topic) .loadConf(Map.of( "topicCompactionStrategyClassName", ServiceUnitStateCompactionStrategy.class.getName())) .create(); + for(var etr : tableview.entrySet()){ Assert.assertEquals(expected.remove(etr.getKey()), etr.getValue()); if (expected.isEmpty()) { @@ -376,6 +379,7 @@ public void testReadCompactedBeforeCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); @@ -420,6 +424,7 @@ public void testReadEntriesAfterCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); @@ -460,6 +465,7 @@ public void testSeekEarliestAfterCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); @@ -557,6 +563,7 @@ public void testSlowTableviewAfterCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .messageRoutingMode(MessageRoutingMode.SinglePartition) .create(); @@ -627,6 +634,7 @@ public void testBrokerRestartAfterCompaction() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); String key = "key0"; @@ -672,6 +680,7 @@ public void testCompactEmptyTopic() throws Exception { Producer producer = pulsarClient.newProducer(schema) .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) .enableBatching(true) .create(); @@ -733,7 +742,9 @@ public void testWholeBatchCompactedOut() throws Exception { public void testCompactionWithLastDeletedKey() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; - Producer producer = pulsarClient.newProducer(schema).topic(topic).enableBatching(true) + Producer producer = pulsarClient.newProducer(schema).topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) + .enableBatching(true) .messageRoutingMode(MessageRoutingMode.SinglePartition).create(); pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); @@ -761,7 +772,9 @@ public void testCompactionWithLastDeletedKey() throws Exception { public void testEmptyCompactionLedger() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; - Producer producer = pulsarClient.newProducer(schema).topic(topic).enableBatching(true) + Producer producer = pulsarClient.newProducer(schema).topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) + .enableBatching(true) .messageRoutingMode(MessageRoutingMode.SinglePartition).create(); pulsarClient.newConsumer(schema).topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); @@ -791,7 +804,8 @@ public void testAllEmptyCompactionLedger() throws Exception { final int messages = 10; // 1.create producer and publish message to the topic. - ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic); + ProducerBuilder builder = pulsarClient.newProducer(schema) + .compressionType(MSG_COMPRESSION_TYPE).topic(topic); builder.batchingMaxMessages(messages / 5); Producer producer = builder.create(); @@ -828,6 +842,7 @@ public void testCompactMultipleTimesWithoutEmptyMessage() // 1.create producer and publish message to the topic. ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic); + builder.compressionType(MSG_COMPRESSION_TYPE); builder.enableBatching(true); @@ -876,6 +891,7 @@ public void testReadUnCompacted() // 1.create producer and publish message to the topic. ProducerBuilder builder = pulsarClient.newProducer(schema).topic(topic); + builder.compressionType(MSG_COMPRESSION_TYPE); builder.batchingMaxMessages(messages / 5); Producer producer = builder.create(); From 0a3a02bb4edb5d36ee3a09766c3c3d671a9044f3 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sat, 15 Apr 2023 13:42:07 +0800 Subject: [PATCH 003/494] [fix][broker] Fix avoid future of clear delayed message can't complete (#20075) (cherry picked from commit 6152cc82ac8a344cfc2539c56f15cec6f4a1cff7) --- .../BucketDelayedDeliveryTrackerFactory.java | 28 +++++++++++ .../pulsar/broker/delayed/bucket/Bucket.java | 3 +- .../bucket/BucketDelayedDeliveryTracker.java | 4 +- ...PersistentDispatcherMultipleConsumers.java | 17 +++---- .../service/persistent/PersistentTopic.java | 26 +++++----- .../persistent/BucketDelayedDeliveryTest.java | 48 +++++++++++++++++++ 6 files changed, 102 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java index f0feb8b27d6a1..6a00bfd199584 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/BucketDelayedDeliveryTrackerFactory.java @@ -21,13 +21,19 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; +import org.apache.pulsar.common.util.FutureUtil; public class BucketDelayedDeliveryTrackerFactory implements DelayedDeliveryTrackerFactory { @@ -72,6 +78,28 @@ public DelayedDeliveryTracker newTracker(PersistentDispatcherMultipleConsumers d delayedDeliveryMaxIndexesPerBucketSnapshotSegment, delayedDeliveryMaxNumBuckets); } + /** + * Clean up residual snapshot data. + * If tracker has not been created or has been closed, then we can't clean up the snapshot with `tracker.clear`, + * this method can clean up the residual snapshots without creating a tracker. + */ + public CompletableFuture cleanResidualSnapshots(ManagedCursor cursor) { + Map cursorProperties = cursor.getCursorProperties(); + List> futures = new ArrayList<>(); + FutureUtil.Sequencer sequencer = FutureUtil.Sequencer.create(); + cursorProperties.forEach((k, v) -> { + if (k != null && v != null && k.startsWith(BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX)) { + CompletableFuture future = sequencer.sequential(() -> { + return cursor.removeCursorProperty(k) + .thenCompose(__ -> bucketSnapshotStorage.deleteBucketSnapshot(Long.parseLong(v))); + }); + futures.add(future); + } + }); + + return FutureUtil.waitForAll(futures); + } + @Override public void close() throws Exception { if (bucketSnapshotStorage != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 5b7023be5034e..4d7d3aa512be6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -18,8 +18,8 @@ */ package org.apache.pulsar.broker.delayed.bucket; -import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.bookkeeper.mledger.util.Futures.executeWithRetry; +import static org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker.DELAYED_BUCKET_KEY_PREFIX; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,7 +41,6 @@ @AllArgsConstructor abstract class Bucket { - static final String DELAYED_BUCKET_KEY_PREFIX = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket"; static final String DELIMITER = "_"; static final int MaxRetryTimes = 3; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index a34bd51af98e4..6ead1e207b0ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELAYED_BUCKET_KEY_PREFIX; +import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; import static org.apache.pulsar.broker.delayed.bucket.Bucket.MaxRetryTimes; import com.google.common.annotations.VisibleForTesting; @@ -66,6 +66,8 @@ @ThreadSafe public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker { + public static final String DELAYED_BUCKET_KEY_PREFIX = CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket"; + static final CompletableFuture NULL_LONG_PROMISE = CompletableFuture.completedFuture(null); static final int AsyncOperationTimeoutSeconds = 60; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 4ac755860fc7b..c60b4562bf111 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -49,6 +49,7 @@ import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; +import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.InMemoryDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.bucket.BucketDelayedDeliveryTracker; import org.apache.pulsar.broker.service.AbstractDispatcherMultipleConsumers; @@ -1098,19 +1099,15 @@ public CompletableFuture clearDelayedMessages() { return CompletableFuture.completedFuture(null); } - if (delayedDeliveryTracker.isEmpty() && topic.getBrokerService() - .getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory) { - synchronized (this) { - if (delayedDeliveryTracker.isEmpty()) { - delayedDeliveryTracker = Optional - .of(topic.getBrokerService().getDelayedDeliveryTrackerFactory().newTracker(this)); - } - } - } - if (delayedDeliveryTracker.isPresent()) { return this.delayedDeliveryTracker.get().clear(); } else { + DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory = + topic.getBrokerService().getDelayedDeliveryTrackerFactory(); + if (delayedDeliveryTrackerFactory instanceof BucketDelayedDeliveryTrackerFactory + bucketDelayedDeliveryTrackerFactory) { + return bucketDelayedDeliveryTrackerFactory.cleanResidualSnapshots(cursor); + } return CompletableFuture.completedFuture(null); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 0f5e60439810f..fe181bb1c01f2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -81,6 +81,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; +import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateCompactionStrategy; import org.apache.pulsar.broker.namespace.NamespaceService; @@ -1169,15 +1170,21 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, } Dispatcher dispatcher = persistentSubscription.getDispatcher(); - final Dispatcher temporaryDispatcher; if (dispatcher == null) { - log.info("[{}][{}] Dispatcher is null, try to create temporary dispatcher to clear delayed message", topic, - subscriptionName); - dispatcher = temporaryDispatcher = - new PersistentDispatcherMultipleConsumers(this, persistentSubscription.cursor, - persistentSubscription); - } else { - temporaryDispatcher = null; + DelayedDeliveryTrackerFactory delayedDeliveryTrackerFactory = + brokerService.getDelayedDeliveryTrackerFactory(); + if (delayedDeliveryTrackerFactory instanceof BucketDelayedDeliveryTrackerFactory + bucketDelayedDeliveryTrackerFactory) { + ManagedCursor cursor = persistentSubscription.getCursor(); + bucketDelayedDeliveryTrackerFactory.cleanResidualSnapshots(cursor).whenComplete((__, ex) -> { + if (ex != null) { + unsubscribeFuture.completeExceptionally(ex); + } else { + asyncDeleteCursor(subscriptionName, unsubscribeFuture); + } + }); + } + return; } dispatcher.clearDelayedMessages().whenComplete((__, ex) -> { @@ -1186,9 +1193,6 @@ private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, } else { asyncDeleteCursor(subscriptionName, unsubscribeFuture); } - if (temporaryDispatcher != null) { - temporaryDispatcher.close(); - } }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java index 5480a2e7a704a..0a82b2b4c3cb0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/BucketDelayedDeliveryTest.java @@ -305,4 +305,52 @@ public void testBucketDelayedIndexMetrics() throws Exception { assertTrue(namespaceMetric.isPresent()); assertEquals(6, namespaceMetric.get().value); } + + @Test + public void testDelete() throws Exception { + String topic = BrokerTestUtil.newUniqueName("persistent://public/default/testDelete"); + + @Cleanup + Consumer c1 = pulsarClient.newConsumer(Schema.STRING) + .topic(topic) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topic) + .create(); + + for (int i = 0; i < 1000; i++) { + producer.newMessage() + .value("msg") + .deliverAfter(1, TimeUnit.HOURS) + .send(); + } + + Dispatcher dispatcher = pulsar.getBrokerService().getTopicReference(topic).get().getSubscription("sub").getDispatcher(); + Awaitility.await().untilAsserted(() -> Assert.assertEquals(dispatcher.getNumberOfDelayedMessages(), 1000)); + + Map cursorProperties = + ((PersistentDispatcherMultipleConsumers) dispatcher).getCursor().getCursorProperties(); + List bucketIds = cursorProperties.entrySet().stream() + .filter(x -> x.getKey().startsWith(CURSOR_INTERNAL_PROPERTY_PREFIX + "delayed.bucket")).map( + x -> Long.valueOf(x.getValue())).toList(); + + assertTrue(bucketIds.size() > 0); + + admin.topics().delete(topic, true); + + for (Long bucketId : bucketIds) { + try { + LedgerHandle ledgerHandle = + pulsarTestContext.getBookKeeperClient() + .openLedger(bucketId, BookKeeper.DigestType.CRC32C, new byte[]{}); + Assert.fail("Should fail"); + } catch (BKException.BKNoSuchLedgerExistsException e) { + // ignore it + } + } + } } From d05871213adc351d4c718c2a6fb0909b01d279ff Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sun, 16 Apr 2023 16:05:44 +0800 Subject: [PATCH 004/494] [fix][broker] Ensure previous delayed index be removed from snapshotSegmentLastIndexTable & Make load operate asynchronous (#20086) (cherry picked from commit c4aec6661e795c46181dc1fa79282065fa875768) --- .../bucket/BucketDelayedDeliveryTracker.java | 106 ++++++++++-------- ...PersistentDispatcherMultipleConsumers.java | 13 +-- .../BucketDelayedDeliveryTrackerTest.java | 30 +++-- 3 files changed, 85 insertions(+), 64 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 6ead1e207b0ce..6678c6df254b0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -21,7 +21,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.CURSOR_INTERNAL_PROPERTY_PREFIX; import static org.apache.pulsar.broker.delayed.bucket.Bucket.DELIMITER; -import static org.apache.pulsar.broker.delayed.bucket.Bucket.MaxRetryTimes; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Range; @@ -84,7 +83,7 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private final int maxNumBuckets; - private long numberDelayedMessages; + private volatile long numberDelayedMessages; @Getter @VisibleForTesting @@ -102,6 +101,8 @@ public class BucketDelayedDeliveryTracker extends AbstractDelayedDeliveryTracker private final BucketDelayedMessageIndexStats stats; + private CompletableFuture pendingLoad = null; + public BucketDelayedDeliveryTracker(PersistentDispatcherMultipleConsumers dispatcher, Timer timer, long tickTimeMillis, boolean isDelayedDeliveryDeliverAtTimeStrict, @@ -269,7 +270,7 @@ private void afterCreateImmutableBucket(Pair immu if (ex == null) { immutableBucket.setSnapshotSegments(null); immutableBucket.asyncUpdateSnapshotLength(); - log.info("[{}] Creat bucket snapshot finish, bucketKey: {}", dispatcher.getName(), + log.info("[{}] Create bucket snapshot finish, bucketKey: {}", dispatcher.getName(), immutableBucket.bucketKey()); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.create, @@ -529,17 +530,25 @@ protected long nextDeliveryTime() { } @Override - public synchronized long getNumberOfDelayedMessages() { + public long getNumberOfDelayedMessages() { return numberDelayedMessages; } @Override - public synchronized long getBufferMemoryUsage() { + public long getBufferMemoryUsage() { return this.lastMutableBucket.getBufferMemoryUsage() + sharedBucketPriorityQueue.bytesCapacity(); } @Override public synchronized NavigableSet getScheduledMessages(int maxMessages) { + if (!checkPendingOpDone()) { + if (log.isDebugEnabled()) { + log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.", + dispatcher.getName()); + } + return Collections.emptyNavigableSet(); + } + long cutoffTime = getCutoffTime(); lastMutableBucket.moveScheduledMessageToSharedQueue(cutoffTime, sharedBucketPriorityQueue); @@ -558,6 +567,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa ImmutableBucket bucket = snapshotSegmentLastIndexTable.get(ledgerId, entryId); if (bucket != null && immutableBuckets.asMapOfRanges().containsValue(bucket)) { + // All message of current snapshot segment are scheduled, try load next snapshot segment if (bucket.merging) { log.info("[{}] Skip load to wait for bucket snapshot merge finish, bucketKey:{}", dispatcher.getName(), bucket.bucketKey()); @@ -569,26 +579,19 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa log.debug("[{}] Loading next bucket snapshot segment, bucketKey: {}, nextSegmentEntryId: {}", dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); } - // All message of current snapshot segment are scheduled, load next snapshot segment - // TODO make it asynchronous and not blocking this process - try { - boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone(); - - if (!createFutureDone) { - log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}", - dispatcher.getName(), bucket.bucketKey()); - break; - } - - if (bucket.currentSegmentEntryId == bucket.lastSegmentEntryId) { - immutableBuckets.asMapOfRanges().remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); - bucket.asyncDeleteBucketSnapshot(stats); - continue; - } + boolean createFutureDone = bucket.getSnapshotCreateFuture().orElse(NULL_LONG_PROMISE).isDone(); + if (!createFutureDone) { + log.info("[{}] Skip load to wait for bucket snapshot create finish, bucketKey:{}", + dispatcher.getName(), bucket.bucketKey()); + break; + } - long loadStartTime = System.currentTimeMillis(); - stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load); - bucket.asyncLoadNextBucketSnapshotEntry().thenAccept(indexList -> { + long loadStartTime = System.currentTimeMillis(); + stats.recordTriggerEvent(BucketDelayedMessageIndexStats.Type.load); + CompletableFuture loadFuture = pendingLoad = bucket.asyncLoadNextBucketSnapshotEntry() + .thenAccept(indexList -> { + synchronized (BucketDelayedDeliveryTracker.this) { + this.snapshotSegmentLastIndexTable.remove(ledgerId, entryId); if (CollectionUtils.isEmpty(indexList)) { immutableBuckets.asMapOfRanges() .remove(Range.closed(bucket.startLedgerId, bucket.endLedgerId)); @@ -603,31 +606,36 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), index.getEntryId()); } - }).whenComplete((__, ex) -> { - if (ex != null) { - // Back bucket state - bucket.setCurrentSegmentEntryId(preSegmentEntryId); - - log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}", - dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex); - - stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load); - } else { - log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", - dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1); - - stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load, - System.currentTimeMillis() - loadStartTime); + } + }).whenComplete((__, ex) -> { + if (ex != null) { + // Back bucket state + bucket.setCurrentSegmentEntryId(preSegmentEntryId); + + log.error("[{}] Failed to load bucket snapshot segment, bucketKey: {}, segmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), preSegmentEntryId + 1, ex); + + stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.load); + } else { + log.info("[{}] Load next bucket snapshot segment finish, bucketKey: {}, segmentEntryId: {}", + dispatcher.getName(), bucket.bucketKey(), + (preSegmentEntryId == bucket.lastSegmentEntryId) ? "-1" : preSegmentEntryId + 1); + + stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.load, + System.currentTimeMillis() - loadStartTime); + } + synchronized (this) { + if (timeout != null) { + timeout.cancel(); } - }).get(AsyncOperationTimeoutSeconds * (MaxRetryTimes + 1), TimeUnit.SECONDS); - } catch (Exception e) { - // Ignore exception to reload this segment on the next schedule. - log.error("[{}] An exception occurs when load next bucket snapshot, bucketKey:{}", - dispatcher.getName(), bucket.bucketKey(), e); + timeout = timer.newTimeout(this, tickTimeMillis, TimeUnit.MILLISECONDS); + } + }); + + if (!checkPendingOpDone() || loadFuture.isCompletedExceptionally()) { break; } } - snapshotSegmentLastIndexTable.remove(ledgerId, entryId); positions.add(new PositionImpl(ledgerId, entryId)); @@ -643,6 +651,14 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa return positions; } + private synchronized boolean checkPendingOpDone() { + if (pendingLoad == null || pendingLoad.isDone()) { + pendingLoad = null; + return true; + } + return false; + } + @Override public boolean shouldPauseAllDeliveries() { return false; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index c60b4562bf111..81adda053e8cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -46,7 +46,6 @@ import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.BucketDelayedDeliveryTrackerFactory; import org.apache.pulsar.broker.delayed.DelayedDeliveryTracker; import org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory; @@ -1089,7 +1088,7 @@ protected synchronized boolean shouldPauseDeliveryForDelayTracker() { } @Override - public synchronized long getNumberOfDelayedMessages() { + public long getNumberOfDelayedMessages() { return delayedDeliveryTracker.map(DelayedDeliveryTracker::getNumberOfDelayedMessages).orElse(0L); } @@ -1169,15 +1168,7 @@ public PersistentTopic getTopic() { public long getDelayedTrackerMemoryUsage() { - if (delayedDeliveryTracker.isEmpty()) { - return 0; - } - - if (delayedDeliveryTracker.get() instanceof AbstractDelayedDeliveryTracker) { - return delayedDeliveryTracker.get().getBufferMemoryUsage(); - } - - return 0; + return delayedDeliveryTracker.map(DelayedDeliveryTracker::getBufferMemoryUsage).orElse(0L); } public Map getBucketDelayedIndexStats() { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java index 95234d688f6a0..39b3992fbd195 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTrackerTest.java @@ -39,6 +39,7 @@ import java.util.NavigableSet; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -197,9 +198,11 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { }); assertTrue(tracker.hasMessageAvailable()); - Set scheduledMessages = tracker.getScheduledMessages(100); - - assertEquals(scheduledMessages.size(), 1); + Set scheduledMessages = new TreeSet<>(); + Awaitility.await().untilAsserted(() -> { + scheduledMessages.addAll(tracker.getScheduledMessages(100)); + assertEquals(scheduledMessages.size(), 1); + }); tracker.addMessage(101, 101, 101 * 10); @@ -216,12 +219,15 @@ public void testRecoverSnapshot(BucketDelayedDeliveryTracker tracker) { clockTime.set(100 * 10); assertTrue(tracker2.hasMessageAvailable()); - scheduledMessages = tracker2.getScheduledMessages(70); + Set scheduledMessages2 = new TreeSet<>(); - assertEquals(scheduledMessages.size(), 70); + Awaitility.await().untilAsserted(() -> { + scheduledMessages2.addAll(tracker2.getScheduledMessages(70)); + assertEquals(scheduledMessages2.size(), 70); + }); int i = 31; - for (PositionImpl scheduledMessage : scheduledMessages) { + for (PositionImpl scheduledMessage : scheduledMessages2) { assertEquals(scheduledMessage, PositionImpl.get(i, i)); i++; } @@ -298,7 +304,11 @@ public void testMergeSnapshot(final BucketDelayedDeliveryTracker tracker) { clockTime.set(110 * 10); - NavigableSet scheduledMessages = tracker2.getScheduledMessages(110); + NavigableSet scheduledMessages = new TreeSet<>(); + Awaitility.await().untilAsserted(() -> { + scheduledMessages.addAll(tracker2.getScheduledMessages(110)); + assertEquals(scheduledMessages.size(), 110); + }); for (int i = 1; i <= 110; i++) { PositionImpl position = scheduledMessages.pollFirst(); assertEquals(position, PositionImpl.get(i, i)); @@ -370,7 +380,11 @@ public void testWithBkException(final BucketDelayedDeliveryTracker tracker) { assertEquals(tracker2.getScheduledMessages(100).size(), 0); - assertEquals(tracker2.getScheduledMessages(100).size(), delayedMessagesInSnapshotValue); + Set scheduledMessages = new TreeSet<>(); + Awaitility.await().untilAsserted(() -> { + scheduledMessages.addAll(tracker2.getScheduledMessages(100)); + assertEquals(scheduledMessages.size(), delayedMessagesInSnapshotValue); + }); assertTrue(mockBucketSnapshotStorage.createExceptionQueue.isEmpty()); assertTrue(mockBucketSnapshotStorage.getMetaDataExceptionQueue.isEmpty()); From a92defcc88d2d80e6d75a5425eb5f734fe0a5f68 Mon Sep 17 00:00:00 2001 From: xiaolong ran Date: Tue, 18 Apr 2023 09:45:19 +0800 Subject: [PATCH 005/494] [improve] [broker] Fix broker restart logic (#20113) (cherry picked from commit 092819b5c4a68dbc39d9a650d5370af9455e805d) --- .../java/org/apache/pulsar/broker/PulsarService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index aad7d32f732c5..5fc9920d0f215 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -467,6 +467,12 @@ public CompletableFuture closeAsync() { protocolHandlers = null; } + // cancel loadShedding task and shutdown the loadManager executor before shutting down the broker + if (this.loadSheddingTask != null) { + this.loadSheddingTask.cancel(); + } + executorServicesShutdown.shutdown(loadManagerExecutor); + List> asyncCloseFutures = new ArrayList<>(); if (this.brokerService != null) { CompletableFuture brokerCloseFuture = this.brokerService.closeAsync(); @@ -501,12 +507,6 @@ public CompletableFuture closeAsync() { this.leaderElectionService = null; } - // cancel loadShedding task and shutdown the loadManager executor before shutting down the broker - if (this.loadSheddingTask != null) { - this.loadSheddingTask.cancel(); - } - executorServicesShutdown.shutdown(loadManagerExecutor); - if (adminClient != null) { adminClient.close(); adminClient = null; From ad831d159ed649dda6cdc554cd77621f5bc63ee4 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Mon, 17 Apr 2023 22:57:48 -0700 Subject: [PATCH 006/494] [improve][broker] Fix ServiceUnitStateCompactionStrategy to cover fast-forward cursor behavior after compaction (#20110) Master Issue: https://github.com/apache/pulsar/issues/16691 ### Motivation Raising a PR to implement: https://github.com/apache/pulsar/issues/16691 After the compaction, the cursor can fast-forward to the compacted horizon when a large number of messages are compacted before the next read. Hence, ServiceUnitStateCompactionStrategy also needs to cover this case. Currently, the existing and slow(their states are far behind) tableviews with ServiceUnitStateCompactionStrategy could not accept those compacted messages. In the load balance extension context, this means the ownership data could be inconsistent among brokers. ### Modifications This PR - fixes ServiceUnitStateCompactionStrategy to accept the state data if its version is bigger than the current version +1. - (minor fix) does not repeatedly update the replication_clusters in the policies when creating the system namespace. This update redundantly triggers ZK watchers when restarting brokers. - sets closeWithoutWaitingClientDisconnect=true, upon unload(following the same setting as the modular LM's) (cherry picked from commit 6cfa4683a44e7cce39fa6cb70e0fe1fb3d5eae56) --- .../pulsar/PulsarClusterMetadataSetup.java | 17 +++- .../channel/ServiceUnitStateChannelImpl.java | 2 +- .../ServiceUnitStateCompactionStrategy.java | 10 ++- .../StrategicTwoPhaseCompactor.java | 2 +- .../ExtensibleLoadManagerImplTest.java | 2 + ...erviceUnitStateCompactionStrategyTest.java | 4 + .../ServiceUnitStateCompactionTest.java | 79 +++++++++++++++++++ .../pulsar/client/impl/TableViewImpl.java | 7 ++ 8 files changed, 114 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index 0badbda1afdfd..9b757c55ccd1d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -385,10 +385,19 @@ static void createNamespaceIfAbsent(PulsarResources resources, NamespaceName nam namespaceResources.createPolicies(namespaceName, policies); } else { log.info("Namespace {} already exists.", namespaceName); - namespaceResources.setPolicies(namespaceName, policies -> { - policies.replication_clusters.add(cluster); - return policies; - }); + var replicaClusterFound = false; + var policiesOptional = namespaceResources.getPolicies(namespaceName); + if (policiesOptional.isPresent() && policiesOptional.get().replication_clusters.contains(cluster)) { + replicaClusterFound = true; + } + if (!replicaClusterFound) { + namespaceResources.setPolicies(namespaceName, policies -> { + policies.replication_clusters.add(cluster); + return policies; + }); + log.info("Updated namespace:{} policies. Added the replication cluster:{}", + namespaceName, cluster); + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 68c6440e68e4b..b4c4e7fd5d42a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -819,7 +819,7 @@ private CompletableFuture closeServiceUnit(String serviceUnit) { NamespaceBundle bundle = getNamespaceBundle(serviceUnit); return pulsar.getBrokerService().unloadServiceUnit( bundle, - false, + true, pulsar.getConfig().getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS) .thenApply(numUnloadedTopics -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index ceb3ea3e9cb6c..72b05b5cd62c8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -52,9 +52,13 @@ public boolean shouldKeepLeft(ServiceUnitStateData from, ServiceUnitStateData to return false; } - // Skip the compaction case where from = null and to.versionId > 1 - if (from != null && from.versionId() + 1 != to.versionId()) { - return true; + if (from != null) { + if (from.versionId() == Long.MAX_VALUE && to.versionId() == Long.MIN_VALUE) { // overflow + } else if (from.versionId() >= to.versionId()) { + return true; + } else if (from.versionId() < to.versionId() - 1) { // Compacted + return false; + } // else from.versionId() == to.versionId() - 1 // continue to check further } if (to.force()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java index 557d4a6580161..a6b0942742763 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java @@ -379,7 +379,7 @@ private CompletableFuture runPhaseTwo( }); }) .thenCompose(v -> { - log.info("Acking ledger id {}", phaseOneResult.firstId); + log.info("Acking ledger id {}", phaseOneResult.lastId); return ((CompactionReaderImpl) reader) .acknowledgeCumulativeAsync( phaseOneResult.lastId, Map.of(COMPACTED_TOPIC_LEDGER_PROPERTY, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 9ab2467d0ef1b..b71eeb4745b87 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -592,6 +592,8 @@ public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Except restartBroker(); pulsar1 = pulsar; setPrimaryLoadManager(); + admin.namespaces().setNamespaceReplicationClusters("public/default", + Sets.newHashSet(this.conf.getClusterName())); var serviceUnitStateChannelPrimaryNew = (ServiceUnitStateChannelImpl) FieldUtils.readDeclaredField(primaryLoadManager, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java index 64964826af652..62de91dab292b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategyTest.java @@ -85,6 +85,10 @@ public void testVersionId(){ new ServiceUnitStateData(Owned, dst, src, 10), new ServiceUnitStateData(Releasing, "broker2", dst, 5))); + assertFalse(strategy.shouldKeepLeft( + new ServiceUnitStateData(Owned, dst, src, 10), + new ServiceUnitStateData(Owned, "broker2", dst, 12))); + } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index 1a69a86f7c6b2..e4f0750a981c9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -27,6 +27,9 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.isValidTransition; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; @@ -49,6 +52,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import org.apache.bookkeeper.client.BookKeeper; import org.apache.commons.lang.reflect.FieldUtils; @@ -69,6 +73,7 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.impl.ReaderImpl; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.policies.data.RetentionPolicies; @@ -628,6 +633,80 @@ public void testSlowTableviewAfterCompaction() throws Exception { } + @Test + public void testSlowReceiveTableviewAfterCompaction() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; + String strategyClassName = "topicCompactionStrategyClassName"; + + pulsarClient.newConsumer(schema) + .topic(topic) + .subscriptionName("sub1") + .readCompacted(true) + .subscribe().close(); + + var tv = pulsar.getClient().newTableViewBuilder(schema) + .topic(topic) + .subscriptionName("slowTV") + .loadConf(Map.of( + strategyClassName, + ServiceUnitStateCompactionStrategy.class.getName())) + .create(); + + // Configure retention to ensue data is retained for reader + admin.namespaces().setRetention("my-property/use/my-ns", + new RetentionPolicies(-1, -1)); + + Producer producer = pulsarClient.newProducer(schema) + .topic(topic) + .compressionType(MSG_COMPRESSION_TYPE) + .enableBatching(true) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + StrategicTwoPhaseCompactor compactor + = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + + var reader = ((CompletableFuture>) FieldUtils + .readDeclaredField(tv, "reader", true)).get(); + var consumer = spy(reader.getConsumer()); + FieldUtils.writeDeclaredField(reader, "consumer", consumer, true); + String bundle = "bundle1"; + final AtomicInteger versionId = new AtomicInteger(0); + final AtomicInteger cnt = new AtomicInteger(1); + int msgAddCount = 1000; // has to be big enough to cover compacted cursor fast-forward. + doAnswer(invocationOnMock -> { + if (cnt.decrementAndGet() == 0) { + var msg = consumer.receiveAsync(); + for (int i = 0; i < msgAddCount; i++) { + producer.newMessage().key(bundle).value( + new ServiceUnitStateData(Owned, "broker" + versionId.incrementAndGet(), true, + versionId.get())).send(); + } + compactor.compact(topic, strategy).join(); + return msg; + } + // Call the real method + reset(consumer); + return consumer.receiveAsync(); + }).when(consumer).receiveAsync(); + producer.newMessage().key(bundle).value( + new ServiceUnitStateData(Owned, "broker", true, + versionId.incrementAndGet())).send(); + producer.newMessage().key(bundle).value( + new ServiceUnitStateData(Owned, "broker" + versionId.incrementAndGet(), true, + versionId.get())).send(); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted( + () -> { + var val = tv.get(bundle); + assertNotNull(val); + assertEquals(val.dstBroker(), "broker" + versionId.get()); + } + ); + + producer.close(); + tv.close(); + } + @Test public void testBrokerRestartAfterCompaction() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java index ff5b251ad55ff..77aba7e48cbad 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java @@ -192,6 +192,13 @@ private void handleMessage(Message msg) { if (compactionStrategy != null) { T prev = data.get(key); update = !compactionStrategy.shouldKeepLeft(prev, cur); + if (!update) { + log.info("Skipped the message from topic {}. key={} value={} prev={}", + conf.getTopicName(), + key, + cur, + prev); + } } if (update) { From 43ef457f2927d2e8276e9e8b0e60e9b004ec2a72 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Wed, 19 Apr 2023 01:27:39 +0800 Subject: [PATCH 007/494] [fix][admin] Add javax.xml.bind:jaxb-api to shade (#20106) Signed-off-by: nodece --- pulsar-client-admin-shaded/pom.xml | 5 +++++ pulsar-client-all/pom.xml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index ae92f1c7a0d85..cdf74cb8eb871 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -131,6 +131,7 @@ com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* javax.ws.rs:* + javax.xml.bind:jaxb-api jakarta.annotation:* org.glassfish.hk2*:* io.grpc:* @@ -230,6 +231,10 @@ javax.annotation org.apache.pulsar.shade.javax.annotation + + javax.xml.bind + org.apache.pulsar.shade.javax.xml.bind + jersey org.apache.pulsar.shade.jersey diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 20be552aed125..5bdc97c362480 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -182,6 +182,7 @@ com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* javax.ws.rs:* + javax.xml.bind:jaxb-api jakarta.annotation:* org.glassfish.hk2*:* io.grpc:* @@ -285,6 +286,10 @@ javax.annotation org.apache.pulsar.shade.javax.annotation + + javax.xml.bind + org.apache.pulsar.shade.javax.xml.bind + jersey org.apache.pulsar.shade.jersey From 13eb2a75f3dc72bed21fdeb4fb0f211a7a0dd3fb Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Wed, 19 Apr 2023 03:01:31 +0800 Subject: [PATCH 008/494] [improve] [broker] Upgrade the BookKeeper dependency to 4.16.1 (#20127) --- .../server/src/assemble/LICENSE.bin.txt | 56 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 6 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 30 +++++----- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index c321e54de2e01..c5e69f5cde97a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -346,34 +346,34 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-proto-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-server-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.0.jar - - org.apache.bookkeeper-circe-checksum-4.16.0.jar - - org.apache.bookkeeper-cpu-affinity-4.16.0.jar - - org.apache.bookkeeper-statelib-4.16.0.jar - - org.apache.bookkeeper-stream-storage-api-4.16.0.jar - - org.apache.bookkeeper-stream-storage-common-4.16.0.jar - - org.apache.bookkeeper-stream-storage-java-client-4.16.0.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.16.0.jar - - org.apache.bookkeeper-stream-storage-proto-4.16.0.jar - - org.apache.bookkeeper-stream-storage-server-4.16.0.jar - - org.apache.bookkeeper-stream-storage-service-api-4.16.0.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.16.0.jar - - org.apache.bookkeeper.http-http-server-4.16.0.jar - - org.apache.bookkeeper.http-vertx-http-server-4.16.0.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.0.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.0.jar - - org.apache.distributedlog-distributedlog-common-4.16.0.jar - - org.apache.distributedlog-distributedlog-core-4.16.0-tests.jar - - org.apache.distributedlog-distributedlog-core-4.16.0.jar - - org.apache.distributedlog-distributedlog-protocol-4.16.0.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.0.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.0.jar - - org.apache.bookkeeper-native-io-4.16.0.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.1.jar + - org.apache.bookkeeper-circe-checksum-4.16.1.jar + - org.apache.bookkeeper-cpu-affinity-4.16.1.jar + - org.apache.bookkeeper-statelib-4.16.1.jar + - org.apache.bookkeeper-stream-storage-api-4.16.1.jar + - org.apache.bookkeeper-stream-storage-common-4.16.1.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.1.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.1.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.1.jar + - org.apache.bookkeeper-stream-storage-server-4.16.1.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.1.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.1.jar + - org.apache.bookkeeper.http-http-server-4.16.1.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.1.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.1.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.1.jar + - org.apache.distributedlog-distributedlog-common-4.16.1.jar + - org.apache.distributedlog-distributedlog-core-4.16.1-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.1.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.1.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.1.jar + - org.apache.bookkeeper-native-io-4.16.1.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 0ac08caa1c01d..cf741622c756f 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -390,9 +390,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.16.0.jar - - cpu-affinity-4.16.0.jar - - circe-checksum-4.16.0.jar + - bookkeeper-common-allocator-4.16.1.jar + - cpu-affinity-4.16.1.jar + - circe-checksum-4.16.1.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index 8daa95e3642b8..f960c694c1905 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.16.0 + 4.16.1 3.8.1 1.5.0 1.10.0 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 214a928a74941..ede2ea3999957 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -426,21 +426,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.16.0.jar - - bookkeeper-common-allocator-4.16.0.jar - - bookkeeper-proto-4.16.0.jar - - bookkeeper-server-4.16.0.jar - - bookkeeper-stats-api-4.16.0.jar - - bookkeeper-tools-framework-4.16.0.jar - - circe-checksum-4.16.0.jar - - codahale-metrics-provider-4.16.0.jar - - cpu-affinity-4.16.0.jar - - http-server-4.16.0.jar - - prometheus-metrics-provider-4.16.0.jar - - codahale-metrics-provider-4.16.0.jar - - bookkeeper-slogger-api-4.16.0.jar - - bookkeeper-slogger-slf4j-4.16.0.jar - - native-io-4.16.0.jar + - bookkeeper-common-4.16.1.jar + - bookkeeper-common-allocator-4.16.1.jar + - bookkeeper-proto-4.16.1.jar + - bookkeeper-server-4.16.1.jar + - bookkeeper-stats-api-4.16.1.jar + - bookkeeper-tools-framework-4.16.1.jar + - circe-checksum-4.16.1.jar + - codahale-metrics-provider-4.16.1.jar + - cpu-affinity-4.16.1.jar + - http-server-4.16.1.jar + - prometheus-metrics-provider-4.16.1.jar + - codahale-metrics-provider-4.16.1.jar + - bookkeeper-slogger-api-4.16.1.jar + - bookkeeper-slogger-slf4j-4.16.1.jar + - native-io-4.16.1.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar From 370f6d7af78fa7bd8f92f8116fac4e3cf32adecb Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 18 Apr 2023 20:24:58 -0500 Subject: [PATCH 009/494] [fix][broker] Implement authenticateAsync for AuthenticationProviderList (#20132) PIP: #12105 and #19771 ### Motivation With the implementation of asynchronous authentication in PIP 97, I missed a case in the `AuthenticationProviderList` where we need to implement the `authenticateAsync` methods. This PR is necessary for making the `AuthenticationProviderToken` and the `AuthenticationProviderOpenID` work together, which is necessary for anyone transitioning to `AuthenticationProviderOpenID`. ### Modifications * Implement `AuthenticationListState#authenticateAsync` using a recursive algorithm that first attempts to authenticate the client using the current `authState` and then tries the remaining options. * Implement `AuthenticationProviderList#authenticateAsync` using a recursive algorithm that attempts each provider sequentially. * Add test to `AuthenticationProviderListTest` that exercises this method. It didn't technically fail previously, but it's worth adding. * Add test to `AuthenticationProviderOpenIDIntegrationTest` to cover the exact failures that were causing problems. (cherry picked from commit 58ccf020fc0fa5f7d9b70032ae60c8bf9a3f234f) --- ...ticationProviderOpenIDIntegrationTest.java | 57 +++++++++++ .../AuthenticationProviderList.java | 95 ++++++++++++++++++- .../AuthenticationProviderListTest.java | 24 +++++ 3 files changed, 175 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java index 298492652c0a8..0075d70f599d3 100644 --- a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java @@ -31,6 +31,7 @@ import com.github.tomakehurst.wiremock.WireMockServer; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.impl.DefaultJwtBuilder; +import io.jsonwebtoken.io.Decoders; import io.jsonwebtoken.security.Keys; import java.io.IOException; import java.nio.file.Files; @@ -41,13 +42,19 @@ import java.util.Base64; import java.util.Date; import java.util.HashMap; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; import javax.naming.AuthenticationException; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataCommand; +import org.apache.pulsar.broker.authentication.AuthenticationProvider; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.broker.authentication.AuthenticationState; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; import org.apache.pulsar.common.api.AuthData; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -438,6 +445,56 @@ public void testAuthenticationStateOpenIDForTokenExpiration() throws Exception { assertTrue(state.isExpired()); } + /** + * This test covers the migration scenario where you have both the Token and OpenID providers. It ensures + * both kinds of authentication work. + * @throws Exception + */ + @Test + public void testAuthenticationProviderListStateSuccess() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName(), + AuthenticationProviderToken.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuer); + + // Set up static token + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + // Use public key for validation + String publicKeyStr = AuthTokenUtils.encodeKeyBase64(keyPair.getPublic()); + props.setProperty("tokenPublicKey", publicKeyStr); + // Use private key to generate token + String privateKeyStr = AuthTokenUtils.encodeKeyBase64(keyPair.getPrivate()); + PrivateKey privateKey = AuthTokenUtils.decodePrivateKey(Decoders.BASE64.decode(privateKeyStr), + SignatureAlgorithm.RS256); + String staticToken = AuthTokenUtils.createToken(privateKey, "superuser", Optional.empty()); + + @Cleanup + AuthenticationService service = new AuthenticationService(conf); + AuthenticationProvider provider = service.getAuthenticationProvider("token"); + + // First, authenticate using OIDC + String role = "superuser"; + String oidcToken = generateToken(validJwk, issuer, role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(oidcToken)).get()); + + // Authenticate using the static token + assertEquals("superuser", provider.authenticateAsync(new AuthenticationDataCommand(staticToken)).get()); + + // Use authenticationState to authentication using OIDC + AuthenticationState state1 = service.getAuthenticationProvider("token").newAuthState(null, null, null); + assertNull(state1.authenticateAsync(AuthData.of(oidcToken.getBytes())).get()); + assertEquals(state1.getAuthRole(), role); + + // Use authenticationState to authentication using static token + AuthenticationState state2 = service.getAuthenticationProvider("token").newAuthState(null, null, null); + assertNull(state2.authenticateAsync(AuthData.of(staticToken.getBytes())).get()); + assertEquals(state1.getAuthRole(), role); + } + @Test void ensureRoleClaimForNonSubClaimReturnsRole() throws Exception { AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java index f8a96aa624e12..16d6f9859a0b4 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java @@ -22,6 +22,7 @@ import java.net.SocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; import javax.servlet.http.HttpServletRequest; @@ -76,9 +77,12 @@ static T applyAuthProcessor(List processors, AuthProcessor authF private static class AuthenticationListState implements AuthenticationState { private final List states; - private AuthenticationState authState; + private volatile AuthenticationState authState; AuthenticationListState(List states) { + if (states == null || states.isEmpty()) { + throw new IllegalArgumentException("Authentication state requires at least one state"); + } this.states = states; this.authState = states.get(0); } @@ -96,6 +100,61 @@ public String getAuthRole() throws AuthenticationException { return getAuthState().getAuthRole(); } + @Override + public CompletableFuture authenticateAsync(AuthData authData) { + // First, attempt to authenticate with the current auth state + CompletableFuture authChallengeFuture = new CompletableFuture<>(); + authState + .authenticateAsync(authData) + .whenComplete((authChallenge, ex) -> { + if (ex == null) { + // Current authState is still correct. Just need to return the authChallenge. + authChallengeFuture.complete(authChallenge); + } else { + if (log.isDebugEnabled()) { + log.debug("Authentication failed for auth provider " + authState.getClass() + ": ", ex); + } + authenticateRemainingAuthStates(authChallengeFuture, authData, ex, states.size() - 1); + } + }); + return authChallengeFuture; + } + + private void authenticateRemainingAuthStates(CompletableFuture authChallengeFuture, + AuthData clientAuthData, + Throwable previousException, + int index) { + if (index < 0) { + if (previousException == null) { + previousException = new AuthenticationException("Authentication required"); + } + AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), + "authentication-provider-list", "Authentication required"); + authChallengeFuture.completeExceptionally(previousException); + return; + } + AuthenticationState state = states.get(index); + if (state == authState) { + // Skip the current auth state + authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, null, index - 1); + } else { + state.authenticateAsync(clientAuthData) + .whenComplete((authChallenge, ex) -> { + if (ex == null) { + // Found the correct auth state + authState = state; + authChallengeFuture.complete(authChallenge); + } else { + if (log.isDebugEnabled()) { + log.debug("Authentication failed for auth provider " + + authState.getClass() + ": ", ex); + } + authenticateRemainingAuthStates(authChallengeFuture, clientAuthData, ex, index - 1); + } + }); + } + } + @Override public AuthData authenticate(AuthData authData) throws AuthenticationException { return applyAuthProcessor( @@ -160,6 +219,40 @@ public String getAuthMethodName() { return providers.get(0).getAuthMethodName(); } + @Override + public CompletableFuture authenticateAsync(AuthenticationDataSource authData) { + CompletableFuture roleFuture = new CompletableFuture<>(); + authenticateRemainingAuthProviders(roleFuture, authData, null, providers.size() - 1); + return roleFuture; + } + + private void authenticateRemainingAuthProviders(CompletableFuture roleFuture, + AuthenticationDataSource authData, + Throwable previousException, + int index) { + if (index < 0) { + if (previousException == null) { + previousException = new AuthenticationException("Authentication required"); + } + AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), + "authentication-provider-list", "Authentication required"); + roleFuture.completeExceptionally(previousException); + return; + } + AuthenticationProvider provider = providers.get(index); + provider.authenticateAsync(authData) + .whenComplete((role, ex) -> { + if (ex == null) { + roleFuture.complete(role); + } else { + if (log.isDebugEnabled()) { + log.debug("Authentication failed for auth provider " + provider.getClass() + ": ", ex); + } + authenticateRemainingAuthProviders(roleFuture, authData, ex, index - 1); + } + }); + } + @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { return applyAuthProcessor( diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java index df011412fee85..7793a5c029f2a 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderListTest.java @@ -161,6 +161,30 @@ public void testAuthenticate() throws Exception { testAuthenticate(tokenBB, SUBJECT_B); } + private void testAuthenticateAsync(String token, String expectedSubject) throws Exception { + String actualSubject = authProvider.authenticateAsync(new AuthenticationDataSource() { + @Override + public boolean hasDataFromCommand() { + return true; + } + + @Override + public String getCommandData() { + return token; + } + }).get(); + assertEquals(actualSubject, expectedSubject); + } + + @Test + public void testAuthenticateAsync() throws Exception { + testAuthenticateAsync(tokenAA, SUBJECT_A); + testAuthenticateAsync(tokenAB, SUBJECT_B); + testAuthenticateAsync(tokenBA, SUBJECT_A); + testAuthenticateAsync(tokenBB, SUBJECT_B); + } + + private AuthenticationState newAuthState(String token, String expectedSubject) throws Exception { // Must pass the token to the newAuthState for legacy reasons. AuthenticationState authState = authProvider.newAuthState( From 7d72df306461da76cca2c895064abefcebb456ad Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Wed, 19 Apr 2023 12:31:25 +0800 Subject: [PATCH 010/494] [fix][broker] Revert : Forbid uploading BYTES schema with admin API (#18995) (#20123) --- .../admin/impl/SchemasResourceBase.java | 4 ---- .../broker/admin/AdminApiSchemaTest.java | 11 ---------- .../org/apache/pulsar/schema/SchemaTest.java | 21 +++++++------------ 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java index 91a8140fde12b..0bab772044a6d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/SchemasResourceBase.java @@ -114,10 +114,6 @@ public CompletableFuture deleteSchemaAsync(boolean authoritative, } public CompletableFuture postSchemaAsync(PostSchemaPayload payload, boolean authoritative) { - if (SchemaType.BYTES.name().equals(payload.getType())) { - return CompletableFuture.failedFuture(new RestException(Response.Status.NOT_ACCEPTABLE, - "Do not upload a BYTES schema, because it's the default schema type")); - } return validateOwnershipAndOperationAsync(authoritative, TopicOperation.PRODUCE) .thenCompose(__ -> getSchemaCompatibilityStrategyAsyncWithoutAuth()) .thenCompose(schemaCompatibilityStrategy -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java index e8e582f80b0dc..b37114f180216 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java @@ -185,17 +185,6 @@ private void testSchemaInfoApi(Schema schema, } - @Test - public void testCreateBytesSchema() { - // forbid admin api creating BYTES schema to be consistent with client side - try { - testSchemaInfoApi(Schema.BYTES, "schematest/test/test-BYTES"); - fail("should fail"); - } catch (Exception e) { - assertTrue(e.getMessage().contains("Do not upload a BYTES schema")); - } - } - @Test(dataProvider = "version") public void testPostSchemaCompatibilityStrategy(ApiVersion version) throws PulsarAdminException { String namespace = format("%s%s%s", "schematest", (ApiVersion.V1.equals(version) ? "/" + cluster + "/" : "/"), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 0ec72b2ef470a..7ba4529cdbdf4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -513,17 +513,13 @@ private void testUseAutoConsumeWithSchemalessTopic(SchemaType schema) throws Exc admin.topics().createPartitionedTopic(topic, 2); // set schema - if (!schema.equals(SchemaType.BYTES)) { - // don't upload bytes schema with admin API - // because null schema means the BYTES schema - SchemaInfo schemaInfo = SchemaInfoImpl - .builder() - .schema(new byte[0]) - .name("dummySchema") - .type(schema) - .build(); - admin.schemas().createSchema(topic, schemaInfo); - } + SchemaInfo schemaInfo = SchemaInfoImpl + .builder() + .schema(new byte[0]) + .name("dummySchema") + .type(schema) + .build(); + admin.schemas().createSchema(topic, schemaInfo); Producer producer = pulsarClient .newProducer() @@ -548,8 +544,7 @@ private void testUseAutoConsumeWithSchemalessTopic(SchemaType schema) throws Exc Message message2 = consumer2.receive(); if (schema == SchemaType.BYTES) { assertEquals(schema, message.getReaderSchema().get().getSchemaInfo().getType()); - // default schema of AUTO_CONSUME is BYTES - assertTrue(message2.getReaderSchema().get().toString().contains("BYTES")); + assertEquals(schema, message2.getReaderSchema().get().getSchemaInfo().getType()); } else if (schema == SchemaType.NONE) { // schema NONE is always reported as BYTES assertEquals(SchemaType.BYTES, message.getReaderSchema().get().getSchemaInfo().getType()); From b839536cb05a0e1c737d9fb87edae1e054fd301b Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 19 Apr 2023 10:44:49 -0500 Subject: [PATCH 011/494] [improve] AuthenticationProviderOpenID k8s error logs (#20135) ### Motivation The `AuthenticationProviderOpenID` error logs from the Kubernetes client are not very helpful in certain cases because we only get the error's message and not the error's response body. See https://github.com/kubernetes-client/java/issues/2066 for details on the solution. Here is an example of a problematic error: ``` org.apache.pulsar.broker.authentication.AuthenticationProviderList - Authentication failed for auth provider class org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID: javax.naming.AuthenticationException: Error retrieving OpenID Provider Metadata from Kubernetes API server: at org.apache.pulsar.broker.authentication.oidc.OpenIDProviderMetadataCache$1.onFailure(OpenIDProviderMetadataCache.java:174) ~[org.apache.pulsar-pulsar-broker-auth-oidc-3.0.0.jar:3.0.0] at io.kubernetes.client.openapi.ApiClient$1.onResponse(ApiClient.java:927) ~[io.kubernetes-client-java-api-17.0.2.jar:?] at okhttp3.internal.connection.RealCall$AsyncCall.run(RealCall.kt:519) ~[com.squareup.okhttp3-okhttp-4.9.3.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:833) ~[?:?] ``` When I enable debug logging out of the API Client, I can see: ``` INFO: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"forbidden: User \"system:serviceaccount:michael-test:superuser\" cannot get path \"/.well-known/openid-configuration/\"","reason":"Forbidden","details":{},"code":403} Apr 19, 2023 2:50:25 AM okhttp3.internal.platform.Platform log INFO: <-- END HTTP (246-byte body) 2023-04-19T02:50:25,832+0000 [pulsar-web-40-1] DEBUG ``` (Note: the solution to this problem is to update the `system:service-account-issuer-discovery` `ClusterRole` to include endpoints with trailing slashes. I created https://github.com/kubernetes/kubernetes/issues/117455 to help solve the permission problem in kubernetes.) ### Modifications * Use both the message and the response body when converting a Kubernetes client error into a Pulsar Authentication error. ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: no need for a forked PR (cherry picked from commit c9c99aae0d0ca5f8ac20a326d3d76bbf8602c79c) --- .../apache/pulsar/broker/authentication/oidc/JwksCache.java | 5 +++-- .../authentication/oidc/OpenIDProviderMetadataCache.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java index 12ea7ec6b906d..b5e038342c2e9 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java @@ -130,9 +130,10 @@ private CompletableFuture> getJwksFromKubernetesApiServer() { @Override public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); + // We want the message and responseBody here: https://github.com/kubernetes-client/java/issues/2066. future.completeExceptionally( - new AuthenticationException("Failed to retrieve public key from Kubernetes API server: " - + e.getMessage())); + new AuthenticationException("Failed to retrieve public key from Kubernetes API server. " + + "Message: " + e.getMessage() + " Response body: " + e.getResponseBody())); } @Override diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java index 33d11f35a349a..111399adbd72b 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/OpenIDProviderMetadataCache.java @@ -165,9 +165,10 @@ private CompletableFuture loadOpenIDProviderMetadataForK @Override public void onFailure(ApiException e, int statusCode, Map> responseHeaders) { incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PROVIDER_METADATA); + // We want the message and responseBody here: https://github.com/kubernetes-client/java/issues/2066. future.completeExceptionally(new AuthenticationException( - "Error retrieving OpenID Provider Metadata from Kubernetes API server: " - + e.getMessage())); + "Error retrieving OpenID Provider Metadata from Kubernetes API server. Message: " + + e.getMessage() + " Response body: " + e.getResponseBody())); } @Override From 0df741259505b9189147d72c6f013b2c18f0436c Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 19 Apr 2023 17:42:23 -0500 Subject: [PATCH 012/494] [fix][fn] Supply download auth params when provided for k8s runtime (#20144) PIP: #19849 ### Motivation The new `KubernetesServiceAccountTokenAuthProvider` introduced by #19888 does not configure the authentication for download because there is an unnecessary check that the `getFunctionAuthenticationSpec` is not `null`. Given that we do not use this spec in creating the auth, it is unnecessary to use here. Further, when we construct the function's authentication, we do not have this null check. See: https://github.com/apache/pulsar/blob/ec102fb024a6ea2b195826778300f20e330dff06/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java#L411-L417 ### Modifications * Update the `KubernetesRuntime` to add authentication params to the download command when provided. ### Verifying this change A new test is added. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: Skipping for this trivial PR (cherry picked from commit 4f2b8e21ff5175f09e9e20ca3ad341db59822c86) --- .../runtime/kubernetes/KubernetesRuntime.java | 3 +-- .../kubernetes/KubernetesRuntimeTest.java | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index e206862b68a67..b1df6c098f6e3 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -879,8 +879,7 @@ private List getDownloadCommand(String tenant, String namespace, String // add auth plugin and parameters if necessary if (authenticationEnabled && authConfig != null) { if (isNotBlank(authConfig.getClientAuthenticationPlugin()) - && isNotBlank(authConfig.getClientAuthenticationParameters()) - && instanceConfig.getFunctionAuthenticationSpec() != null) { + && isNotBlank(authConfig.getClientAuthenticationParameters())) { cmd.addAll(Arrays.asList( "--auth-plugin", authConfig.getClientAuthenticationPlugin(), diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 1e8194eed443a..3facd37fc9244 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -861,6 +861,32 @@ public void testCustomKubernetesDownloadCommandsWithAuth() throws Exception { assertTrue(containerCommand.contains(expectedDownloadCommand), "Found:" + containerCommand); } + @Test + public void testCustomKubernetesDownloadCommandsWithAuthWithoutAuthSpec() throws Exception { + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false); + config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false)); + + factory = createKubernetesRuntimeFactory(null, + 10, 1.0, 1.0, Optional.empty(), null, wconfig -> { + wconfig.setAuthenticationEnabled(true); + }, AuthenticationConfig.builder() + .clientAuthenticationPlugin("com.MyAuth") + .clientAuthenticationParameters("{\"authParam1\": \"authParamValue1\"}") + .build()); + + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); + V1StatefulSet spec = container.createStatefulSet(); + String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl + + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}" + + " functions download " + + "--tenant " + TEST_TENANT + + " --namespace " + TEST_NAMESPACE + + " --name " + TEST_NAME + + " --destination-file " + pulsarRootDir + "/" + userJarFile; + String containerCommand = spec.getSpec().getTemplate().getSpec().getContainers().get(0).getCommand().get(2); + assertTrue(containerCommand.contains(expectedDownloadCommand), "Found:" + containerCommand); + } + InstanceConfig createGolangInstanceConfig() { InstanceConfig config = new InstanceConfig(); From 1d1a3ef864a65c995ceda4b7875ed934c2574298 Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Mon, 17 Apr 2023 19:32:13 +0800 Subject: [PATCH 013/494] [improve] [broker] Close temporary open ledger in BookkeeperBucketSnapshotStorage (#20111) (cherry picked from commit b50e8802a5224dd68832e263e7046650771a1a4e) --- .../BookkeeperBucketSnapshotStorage.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index e7d4f9301dd36..9c30ccf1c0b7e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -66,22 +66,41 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - return openLedger(bucketId).thenCompose( - ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0). - thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture snapshotFuture = + getLedgerEntry(ledgerHandle, 0, 0) + .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())); + + snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return snapshotFuture; + }); } @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { - return openLedger(bucketId).thenCompose( - ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, - lastSegmentEntryId).thenApply(this::parseSnapshotSegmentEntries)); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture> parseFuture = + getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) + .thenApply(this::parseSnapshotSegmentEntries); + + parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return parseFuture; + }); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return openLedger(bucketId).thenApply(LedgerHandle::getLength); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture lengthFuture = + CompletableFuture.completedFuture(ledgerHandle.getLength()); + + lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return lengthFuture; + }); } @Override From 49480ea558e647169e8df01bfd2e871a5386e19e Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 19 Apr 2023 16:44:12 +0800 Subject: [PATCH 014/494] [improve][broker] Make timer execute immediately after load index (#20126) (cherry picked from commit 9b723022436cc1a150af765103e8d343679f92ce) --- .../delayed/bucket/BucketDelayedDeliveryTracker.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 6678c6df254b0..ad2fc6fae4c87 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -541,7 +541,7 @@ public long getBufferMemoryUsage() { @Override public synchronized NavigableSet getScheduledMessages(int maxMessages) { - if (!checkPendingOpDone()) { + if (!checkPendingLoadDone()) { if (log.isDebugEnabled()) { log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.", dispatcher.getName()); @@ -628,11 +628,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa if (timeout != null) { timeout.cancel(); } - timeout = timer.newTimeout(this, tickTimeMillis, TimeUnit.MILLISECONDS); + timeout = timer.newTimeout(this, 0, TimeUnit.MILLISECONDS); } }); - if (!checkPendingOpDone() || loadFuture.isCompletedExceptionally()) { + if (!checkPendingLoadDone() || loadFuture.isCompletedExceptionally()) { break; } } @@ -651,7 +651,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa return positions; } - private synchronized boolean checkPendingOpDone() { + private synchronized boolean checkPendingLoadDone() { if (pendingLoad == null || pendingLoad.isDone()) { pendingLoad = null; return true; From ff59240165c73a9c3a3dcca20702ab44b0b18d33 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 20 Apr 2023 08:21:08 +0800 Subject: [PATCH 015/494] [improve][broker] Optimize delayed metadata index bitmap (#20136) (cherry picked from commit 6dc0b0ea38ab9dc410a24b19fd7567b9e013837d) --- .../bucket/BucketDelayedDeliveryTracker.java | 3 ++ .../delayed/bucket/ImmutableBucket.java | 53 +++++++++++-------- .../broker/delayed/bucket/MutableBucket.java | 11 ++-- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index ad2fc6fae4c87..5d1dc2e27033a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -485,6 +485,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List> asyncLoadNextBucketSnapshotEntry(b this.setLastSegmentEntryId(metadataList.size()); this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); List firstScheduleTimestamps = metadataList.stream().map( - SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); + SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); this.setFirstScheduleTimestamps(firstScheduleTimestamps); return nextSnapshotEntryIndex + 1; @@ -139,25 +138,37 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b }); } + /** + * Recover delayed index bit map and message numbers. + * @throws InvalidRoaringFormat invalid bitmap serialization format + */ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, - List segmentMetadata) { - this.delayedIndexBitMap.clear(); - MutableLong numberMessages = new MutableLong(0); - for (int i = startSnapshotIndex; i < segmentMetadata.size(); i++) { - Map bitByteStringMap = segmentMetadata.get(i).getDelayedIndexBitMapMap(); - bitByteStringMap.forEach((leaderId, bitSetString) -> { - boolean exist = this.delayedIndexBitMap.containsKey(leaderId); - RoaringBitmap bitSet = - new ImmutableRoaringBitmap(bitSetString.asReadOnlyByteBuffer()).toRoaringBitmap(); - numberMessages.add(bitSet.getCardinality()); - if (!exist) { - this.delayedIndexBitMap.put(leaderId, bitSet); - } else { - this.delayedIndexBitMap.get(leaderId).or(bitSet); + List segmentMetaList) { + delayedIndexBitMap.clear(); // cleanup dirty bm + final var numberMessages = new MutableLong(0); + for (int i = startSnapshotIndex; i < segmentMetaList.size(); i++) { + for (final var entry : segmentMetaList.get(i).getDelayedIndexBitMapMap().entrySet()) { + final var ledgerId = entry.getKey(); + final var bs = entry.getValue(); + final var sbm = new RoaringBitmap(); + try { + sbm.deserialize(bs.asReadOnlyByteBuffer()); + } catch (IOException e) { + throw new InvalidRoaringFormat(e.getMessage()); } - }); + numberMessages.add(sbm.getCardinality()); + delayedIndexBitMap.compute(ledgerId, (lId, bm) -> { + if (bm == null) { + return sbm; + } + bm.or(sbm); + return bm; + }); + } } - this.setNumberBucketDelayedMessages(numberMessages.getValue()); + // optimize bm + delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize); + setNumberBucketDelayedMessages(numberMessages.getValue()); } CompletableFuture> getRemainSnapshotSegment() { @@ -193,7 +204,7 @@ CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete); } else { log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey); + dispatcherName, bucketId, bucketKey); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete, System.currentTimeMillis() - deleteStartTime); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index e49ebe9606e01..f404d5d02c15a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -116,10 +116,13 @@ Pair createImmutableBucketAndAsyncPersistent( Iterator> iterator = bitMap.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - byte[] array = new byte[entry.getValue().serializedSizeInBytes()]; - entry.getValue().serialize(ByteBuffer.wrap(array)); - segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), ByteString.copyFrom(array)); + final var entry = iterator.next(); + final var lId = entry.getKey(); + final var bm = entry.getValue(); + bm.runOptimize(); + final var array = new byte[bm.serializedSizeInBytes()]; + bm.serialize(ByteBuffer.wrap(array)); + segmentMetadataBuilder.putDelayedIndexBitMap(lId, ByteString.copyFrom(array)); iterator.remove(); } From e1d63990644700bf61b3d7af1ef6d4d62145c2bb Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 20 Apr 2023 09:26:30 +0800 Subject: [PATCH 016/494] [improve][broker] Cache LedgerHandle in BookkeeperBucketSnapshotStorage (#20117) (cherry picked from commit d3fa998aa7c0a7a9452079ef2ff05bccf6b273cf) --- .../BookkeeperBucketSnapshotStorage.java | 52 +++++++++---------- .../BookkeeperBucketSnapshotStorageTest.java | 43 +++++++++++++++ 2 files changed, 69 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 9c30ccf1c0b7e..18a4c322f7b27 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import javax.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; @@ -48,6 +49,8 @@ public class BookkeeperBucketSnapshotStorage implements BucketSnapshotStorage { private final ServiceConfiguration config; private BookKeeper bookKeeper; + private final Map> ledgerHandleFutureCache = new ConcurrentHashMap<>(); + public BookkeeperBucketSnapshotStorage(PulsarService pulsar) { this.pulsar = pulsar; this.config = pulsar.getConfig(); @@ -66,45 +69,30 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture snapshotFuture = - getLedgerEntry(ledgerHandle, 0, 0) - .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())); - - snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return snapshotFuture; - }); + return getLedgerHandle(bucketId).thenCompose(ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0) + .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); } @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture> parseFuture = - getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) - .thenApply(this::parseSnapshotSegmentEntries); - - parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return parseFuture; - }); + return getLedgerHandle(bucketId).thenCompose( + ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) + .thenApply(this::parseSnapshotSegmentEntries)); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture lengthFuture = - CompletableFuture.completedFuture(ledgerHandle.getLength()); - - lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return lengthFuture; - }); + return getLedgerHandle(bucketId).thenCompose( + ledgerHandle -> CompletableFuture.completedFuture(ledgerHandle.getLength())); } @Override public CompletableFuture deleteBucketSnapshot(long bucketId) { + CompletableFuture ledgerHandleFuture = ledgerHandleFutureCache.remove(bucketId); + if (ledgerHandleFuture != null) { + ledgerHandleFuture.whenComplete((lh, ex) -> closeLedger(lh)); + } return deleteLedger(bucketId); } @@ -178,6 +166,18 @@ private CompletableFuture createLedger(String bucketKey, String to return future; } + private CompletableFuture getLedgerHandle(Long ledgerId) { + CompletableFuture ledgerHandleCompletableFuture = + ledgerHandleFutureCache.computeIfAbsent(ledgerId, k -> openLedger(ledgerId)); + // remove future of completed exceptionally + ledgerHandleCompletableFuture.whenComplete((__, ex) -> { + if (ex != null) { + ledgerHandleFutureCache.remove(ledgerId, ledgerHandleCompletableFuture); + } + }); + return ledgerHandleCompletableFuture; + } + private CompletableFuture openLedger(Long ledgerId) { final CompletableFuture future = new CompletableFuture<>(); bookKeeper.asyncOpenLedger( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index a628b58e10d32..7cb6b8d5865bb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -204,4 +205,46 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted Assert.assertTrue(bucketSnapshotLength > 0L); } + @Test + public void testConcurrencyGet() throws ExecutionException, InterruptedException { + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + .setMinScheduleTimestamp(System.currentTimeMillis()) + .setMaxScheduleTimestamp(System.currentTimeMillis()) + .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); + + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + .addMetadataList(segmentMetadata) + .build(); + List bucketSnapshotSegments = new ArrayList<>(); + + long timeMillis = System.currentTimeMillis(); + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis).build(); + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + bucketSnapshotSegments.add(snapshotSegment); + bucketSnapshotSegments.add(snapshotSegment); + + CompletableFuture future = + bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, + bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); + Long bucketId = future.get(); + Assert.assertNotNull(bucketId); + + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + CompletableFuture future0 = CompletableFuture.runAsync(() -> { + List list = + bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join(); + Assert.assertTrue(list.size() > 0); + }); + futures.add(future0); + } + + FutureUtil.waitForAll(futures).join(); + } + } From e27abe9e128fb71b65ffe06417574c9a7f3facbd Mon Sep 17 00:00:00 2001 From: ran Date: Thu, 20 Apr 2023 16:18:06 +0800 Subject: [PATCH 017/494] [fix][broker] Fix entry filter feature for the non-persistent topic (#20141) (cherry picked from commit 575cf2331dd3ac0048923487c6c7904eda4301e6) --- .../nonpersistent/NonPersistentTopic.java | 5 ++- .../service/plugin/FilterEntryTest.java | 15 +++++-- .../broker/stats/SubscriptionStatsTest.java | 42 ++++++++++--------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 317b8df6b9a82..33258b06726b5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -26,7 +26,7 @@ import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -199,7 +199,8 @@ public void publishMessage(ByteBuf data, PublishContext callback) { // entry internally retains data so, duplicateBuffer should be release here duplicateBuffer.release(); if (subscription.getDispatcher() != null) { - subscription.getDispatcher().sendMessages(Collections.singletonList(entry)); + // Dispatcher needs to call the set method to support entry filter feature. + subscription.getDispatcher().sendMessages(Arrays.asList(entry)); } else { // it happens when subscription is created but dispatcher is not created as consumer is not added // yet diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java index 4b9d91fbde219..b868858646c50 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.EntryFilterSupport; +import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -286,10 +287,16 @@ public void testFilter() throws Exception { } + @DataProvider(name = "topicProvider") + public Object[][] topicProvider() { + return new Object[][]{ + {"persistent://prop/ns-abc/topic" + UUID.randomUUID()}, + {"non-persistent://prop/ns-abc/topic" + UUID.randomUUID()}, + }; + } - @Test - public void testFilteredMsgCount() throws Throwable { - String topic = "persistent://prop/ns-abc/topic" + UUID.randomUUID(); + @Test(dataProvider = "topicProvider") + public void testFilteredMsgCount(String topic) throws Throwable { String subName = "sub"; try (Producer producer = pulsarClient.newProducer(Schema.STRING) @@ -298,7 +305,7 @@ public void testFilteredMsgCount() throws Throwable { .subscriptionName(subName).subscribe()) { // mock entry filters - PersistentSubscription subscription = (PersistentSubscription) pulsar.getBrokerService() + Subscription subscription = pulsar.getBrokerService() .getTopicReference(topic).get().getSubscription(subName); Dispatcher dispatcher = subscription.getDispatcher(); Field field = EntryFilterSupport.class.getDeclaredField("entryFilters"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java index bf9c1d540bf87..d5e0066a86f15 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java @@ -211,21 +211,22 @@ public void testSubscriptionStats(final String topic, final String subName, bool hasFilterField.set(dispatcher, true); } - for (int i = 0; i < 100; i++) { - producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send(); - } - for (int i = 0; i < 100; i++) { + int rejectedCount = 100; + int acceptCount = 100; + int scheduleCount = 100; + for (int i = 0; i < rejectedCount; i++) { producer.newMessage().property("REJECT", " ").value(UUID.randomUUID().toString()).send(); } - for (int i = 0; i < 100; i++) { + for (int i = 0; i < acceptCount; i++) { + producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send(); + } + for (int i = 0; i < scheduleCount; i++) { producer.newMessage().property("RESCHEDULE", " ").value(UUID.randomUUID().toString()).send(); } - for (;;) { - Message message = consumer.receive(10, TimeUnit.SECONDS); - if (message == null) { - break; - } + for (int i = 0; i < acceptCount; i++) { + Message message = consumer.receive(1, TimeUnit.SECONDS); + Assert.assertNotNull(message); consumer.acknowledge(message); } @@ -263,12 +264,12 @@ public void testSubscriptionStats(final String topic, final String subName, bool .mapToDouble(m-> m.value).sum(); if (setFilter) { - Assert.assertEquals(filterAccepted, 100); - if (isPersistent) { - Assert.assertEquals(filterRejected, 100); - // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount - Assert.assertEquals(throughFilter, filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter); - } + Assert.assertEquals(filterAccepted, acceptCount); + Assert.assertEquals(filterRejected, rejectedCount); + // Only works on the test, if there are some markers, + // the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount + Assert.assertEquals(throughFilter, + filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter); } else { Assert.assertEquals(throughFilter, 0D); Assert.assertEquals(filterAccepted, 0D); @@ -282,19 +283,20 @@ public void testSubscriptionStats(final String topic, final String subName, bool Assert.assertEquals(rescheduledMetrics.size(), 0); } - testSubscriptionStatsAdminApi(topic, subName, setFilter); + testSubscriptionStatsAdminApi(topic, subName, setFilter, acceptCount, rejectedCount); } - private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter) throws Exception { + private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter, + int acceptCount, int rejectedCount) throws Exception { boolean persistent = TopicName.get(topic).isPersistent(); TopicStats topicStats = admin.topics().getStats(topic); SubscriptionStats stats = topicStats.getSubscriptions().get(subName); Assert.assertNotNull(stats); if (setFilter) { - Assert.assertEquals(stats.getFilterAcceptedMsgCount(), 100); + Assert.assertEquals(stats.getFilterAcceptedMsgCount(), acceptCount); if (persistent) { - Assert.assertEquals(stats.getFilterRejectedMsgCount(), 100); + Assert.assertEquals(stats.getFilterRejectedMsgCount(), rejectedCount); // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount Assert.assertEquals(stats.getFilterProcessedMsgCount(), stats.getFilterAcceptedMsgCount() + stats.getFilterRejectedMsgCount() From 03ec01d14d418233ce2fba2da7f429665df90de5 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 20 Apr 2023 17:49:27 +0800 Subject: [PATCH 018/494] [fix][client] Move MessageIdAdv to the pulsar-common module (#20139) (cherry picked from commit 99a68e40c9dcc0771ddfd73321cc967be6433801) --- .../pulsar/client/api/TopicMessageId.java | 82 +------------------ .../PulsarClientImplementationBinding.java | 3 + .../pulsar/client/impl/ConsumerImpl.java | 2 +- .../pulsar/client/impl/MessageIdImpl.java | 4 +- ...PulsarClientImplementationBindingImpl.java | 18 +++- .../client/impl/TopicMessageIdImpl.java | 77 +++++++++++++++-- .../pulsar/client/impl/TopicMessageImpl.java | 3 +- .../client/impl/MessageIdCompareToTest.java | 5 -- .../client/impl/TopicMessageIdImplTest.java | 12 +-- .../pulsar/client/api/MessageIdAdv.java | 0 .../pulsar/client/api/package-info.java | 22 +++++ .../kafka/connect/KafkaConnectSinkTest.java | 4 +- .../pulsar/websocket/ConsumerHandler.java | 6 +- 13 files changed, 133 insertions(+), 105 deletions(-) rename {pulsar-client-api => pulsar-common}/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java (100%) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java index b70267bb0fb8b..4d02a7f4096d6 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/TopicMessageId.java @@ -18,7 +18,7 @@ */ package org.apache.pulsar.client.api; -import java.util.BitSet; +import org.apache.pulsar.client.internal.DefaultImplementation; /** * The MessageId used for a consumer that subscribes multiple topics or partitioned topics. @@ -45,84 +45,6 @@ static TopicMessageId create(String topic, MessageId messageId) { if (messageId instanceof TopicMessageId) { return (TopicMessageId) messageId; } - return new Impl(topic, messageId); - } - - /** - * The simplest implementation of a TopicMessageId interface. - */ - class Impl implements MessageIdAdv, TopicMessageId { - private final String topic; - private final MessageIdAdv messageId; - - public Impl(String topic, MessageId messageId) { - this.topic = topic; - this.messageId = (MessageIdAdv) messageId; - } - - @Override - public byte[] toByteArray() { - return messageId.toByteArray(); - } - - @Override - public String getOwnerTopic() { - return topic; - } - - @Override - public long getLedgerId() { - return messageId.getLedgerId(); - } - - @Override - public long getEntryId() { - return messageId.getEntryId(); - } - - @Override - public int getPartitionIndex() { - return messageId.getPartitionIndex(); - } - - @Override - public int getBatchIndex() { - return messageId.getBatchIndex(); - } - - @Override - public int getBatchSize() { - return messageId.getBatchSize(); - } - - @Override - public BitSet getAckSet() { - return messageId.getAckSet(); - } - - @Override - public MessageIdAdv getFirstChunkMessageId() { - return messageId.getFirstChunkMessageId(); - } - - @Override - public int compareTo(MessageId o) { - return messageId.compareTo(o); - } - - @Override - public boolean equals(Object obj) { - return messageId.equals(obj); - } - - @Override - public int hashCode() { - return messageId.hashCode(); - } - - @Override - public String toString() { - return messageId.toString(); - } + return DefaultImplementation.getDefaultImplementation().newTopicMessageId(topic, messageId); } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java index 875a793023523..8fd05bff265f1 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/internal/PulsarClientImplementationBinding.java @@ -37,6 +37,7 @@ import org.apache.pulsar.client.api.MessagePayloadFactory; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; @@ -252,4 +253,6 @@ static byte[] getBytes(ByteBuffer byteBuffer) { SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, long timestamp, Map propertiesValue); + + TopicMessageId newTopicMessageId(String topic, MessageId messageId); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index cc01609319698..199e8a9ae71b4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2345,7 +2345,7 @@ public CompletableFuture getLastMessageIdAsync() { @Override public CompletableFuture> getLastMessageIdsAsync() { return getLastMessageIdAsync() - .thenApply(msgId -> Collections.singletonList(TopicMessageId.create(topic, msgId))); + .thenApply(msgId -> Collections.singletonList(new TopicMessageIdImpl(topic, (MessageIdAdv) msgId))); } public CompletableFuture internalGetLastMessageIdAsync() { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java index 83ee762578390..8cffba44dc5ca 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java @@ -128,7 +128,7 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName) throw new IOException(e); } - MessageId messageId; + MessageIdAdv messageId; if (idData.hasBatchIndex()) { if (idData.hasBatchSize()) { messageId = new BatchMessageIdImpl(idData.getLedgerId(), idData.getEntryId(), idData.getPartition(), @@ -143,7 +143,7 @@ public static MessageId fromByteArrayWithTopic(byte[] data, TopicName topicName) } if (idData.getPartition() > -1 && topicName != null) { messageId = new TopicMessageIdImpl( - topicName.getPartition(idData.getPartition()).toString(), topicName.toString(), messageId); + topicName.getPartition(idData.getPartition()).toString(), messageId); } return messageId; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java index 1b069c5172dd7..346eb20ef4cc5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImplementationBindingImpl.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.client.impl; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -35,9 +34,11 @@ import org.apache.pulsar.client.api.BatcherBuilder; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessagePayloadFactory; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.api.schema.GenericSchema; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; @@ -387,4 +388,19 @@ public SchemaInfo newSchemaInfoImpl(String name, byte[] schema, SchemaType type, Map propertiesValue) { return new SchemaInfoImpl(name, schema, type, timestamp, propertiesValue); } + + @Override + public TopicMessageId newTopicMessageId(String topic, MessageId messageId) { + final MessageIdAdv messageIdAdv; + if (messageId instanceof MessageIdAdv) { + messageIdAdv = (MessageIdAdv) messageId; + } else { + try { + messageIdAdv = (MessageIdAdv) MessageId.fromByteArray(messageId.toByteArray()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return new TopicMessageIdImpl(topic, messageIdAdv); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java index 189dc1c608379..00fe12b62b194 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java @@ -18,15 +18,27 @@ */ package org.apache.pulsar.client.impl; +import java.util.BitSet; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.TopicMessageId; -public class TopicMessageIdImpl extends TopicMessageId.Impl { +public class TopicMessageIdImpl implements MessageIdAdv, TopicMessageId { - private final String topicName; + private final String ownerTopic; + private final MessageIdAdv msgId; + private final String topicName; // it's never used + public TopicMessageIdImpl(String topic, MessageIdAdv msgId) { + this.ownerTopic = topic; + this.msgId = msgId; + this.topicName = ""; + } + + @Deprecated public TopicMessageIdImpl(String topicPartitionName, String topicName, MessageId messageId) { - super(topicPartitionName, messageId); + this.msgId = (MessageIdAdv) messageId; + this.ownerTopic = topicPartitionName; this.topicName = topicName; } @@ -55,11 +67,66 @@ public MessageId getInnerMessageId() { @Override public boolean equals(Object obj) { - return super.equals(obj); + return msgId.equals(obj); } @Override public int hashCode() { - return super.hashCode(); + return msgId.hashCode(); + } + + @Override + public int compareTo(MessageId o) { + return msgId.compareTo(o); + } + + @Override + public byte[] toByteArray() { + return msgId.toByteArray(); + } + + @Override + public String getOwnerTopic() { + return ownerTopic; + } + + @Override + public long getLedgerId() { + return msgId.getLedgerId(); + } + + @Override + public long getEntryId() { + return msgId.getEntryId(); + } + + @Override + public int getPartitionIndex() { + return msgId.getPartitionIndex(); + } + + @Override + public int getBatchIndex() { + return msgId.getBatchIndex(); + } + + @Override + public int getBatchSize() { + return msgId.getBatchSize(); + } + + @Override + public BitSet getAckSet() { + return msgId.getAckSet(); + } + + @Override + public MessageIdAdv getFirstChunkMessageId() { + return msgId.getFirstChunkMessageId(); + } + + @Override + public String toString() { + return msgId.toString(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java index d24ecbd6aa917..1b6cba2f7234d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageImpl.java @@ -22,6 +22,7 @@ import java.util.Optional; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.api.EncryptionContext; @@ -42,7 +43,7 @@ public class TopicMessageImpl implements Message { this.receivedByconsumer = receivedByConsumer; this.msg = msg; - this.messageId = new TopicMessageIdImpl(topicPartitionName, topicPartitionName, msg.getMessageId()); + this.messageId = new TopicMessageIdImpl(topicPartitionName, (MessageIdAdv) msg.getMessageId()); } /** diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java index 4f0eca6ea4af8..fd81e9d5790ad 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/MessageIdCompareToTest.java @@ -148,15 +148,12 @@ public void testMessageIdImplCompareToTopicMessageId() { MessageIdImpl messageIdImpl = new MessageIdImpl(123L, 345L, 567); TopicMessageIdImpl topicMessageId1 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new BatchMessageIdImpl(123L, 345L, 566, 789)); TopicMessageIdImpl topicMessageId2 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new BatchMessageIdImpl(123L, 345L, 567, 789)); TopicMessageIdImpl topicMessageId3 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new BatchMessageIdImpl(messageIdImpl)); assertTrue(messageIdImpl.compareTo(topicMessageId1) > 0, "Expected to be greater than"); assertTrue(messageIdImpl.compareTo(topicMessageId2) < 0, "Expected to be less than"); @@ -173,11 +170,9 @@ public void testBatchMessageIdImplCompareToTopicMessageId() { BatchMessageIdImpl messageIdImpl3 = new BatchMessageIdImpl(123L, 345L, 567, -1); TopicMessageIdImpl topicMessageId1 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new MessageIdImpl(123L, 345L, 566)); TopicMessageIdImpl topicMessageId2 = new TopicMessageIdImpl( "test-topic-partition-0", - "test-topic", new MessageIdImpl(123L, 345L, 567)); assertTrue(messageIdImpl1.compareTo(topicMessageId1) > 0, "Expected to be greater than"); assertTrue(messageIdImpl1.compareTo(topicMessageId2) > 0, "Expected to be greater than"); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java index d2e2ce9c15c54..daf49f0e7752e 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java @@ -28,9 +28,9 @@ public class TopicMessageIdImplTest { public void hashCodeTest() { MessageIdImpl msgId1 = new MessageIdImpl(0, 0, 0); MessageIdImpl msgId2 = new BatchMessageIdImpl(1, 1, 1, 1); - TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", "topic", msgId1); - TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", "topic2", msgId1); - TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", "topic", msgId2); + TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", msgId1); + TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", msgId1); + TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", msgId2); assertEquals(topicMsgId1.hashCode(), topicMsgId1.hashCode()); assertEquals(topic2MsgId1.hashCode(), topic2MsgId1.hashCode()); @@ -43,9 +43,9 @@ public void hashCodeTest() { public void equalsTest() { MessageIdImpl msgId1 = new MessageIdImpl(0, 0, 0); MessageIdImpl msgId2 = new BatchMessageIdImpl(1, 1, 1, 1); - TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", "topic", msgId1); - TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", "topic2", msgId1); - TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", "topic", msgId2); + TopicMessageIdImpl topicMsgId1 = new TopicMessageIdImpl("topic-partition-1", msgId1); + TopicMessageIdImpl topic2MsgId1 = new TopicMessageIdImpl("topic2-partition-1", msgId1); + TopicMessageIdImpl topicMsgId2 = new TopicMessageIdImpl("topic-partition-2", msgId2); assertEquals(topicMsgId1, topicMsgId1); assertEquals(topicMsgId1, topic2MsgId1); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java b/pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java similarity index 100% rename from pulsar-client-api/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java rename to pulsar-common/src/main/java/org/apache/pulsar/client/api/MessageIdAdv.java diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java b/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java new file mode 100644 index 0000000000000..3f6d1d56e1032 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/api/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Additional helper classes to the pulsar-client-api module. + */ +package org.apache.pulsar.client.api; diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java index 5410e0bb8d664..1100b13b425b4 100644 --- a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSinkTest.java @@ -1572,7 +1572,7 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception { assertNull(ref); ref = KafkaConnectSink.getMessageSequenceRefForBatchMessage( - new TopicMessageIdImpl("topic-0", "topic", new MessageIdImpl(ledgerId, entryId, 0)) + new TopicMessageIdImpl("topic-0", new MessageIdImpl(ledgerId, entryId, 0)) ); assertNull(ref); @@ -1584,7 +1584,7 @@ public void testGetMessageSequenceRefForBatchMessage() throws Exception { assertEquals(ref.getBatchIdx(), batchIdx); ref = KafkaConnectSink.getMessageSequenceRefForBatchMessage( - new TopicMessageIdImpl("topic-0", "topic", new BatchMessageIdImpl(ledgerId, entryId, 0, batchIdx)) + new TopicMessageIdImpl("topic-0", new BatchMessageIdImpl(ledgerId, entryId, 0, batchIdx)) ); assertEquals(ref.getLedgerId(), ledgerId); diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java index 579b423339911..c988fd1e70ce3 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/ConsumerHandler.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.DeadLetterPolicy; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.AlreadyClosedException; @@ -45,6 +46,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.impl.ConsumerBuilderImpl; +import org.apache.pulsar.client.impl.TopicMessageIdImpl; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.websocket.data.ConsumerCommand; @@ -293,8 +295,8 @@ private void checkResumeReceive() { private void handleAck(ConsumerCommand command) throws IOException { // We should have received an ack - TopicMessageId msgId = TopicMessageId.create(topic.toString(), - MessageId.fromByteArray(Base64.getDecoder().decode(command.messageId))); + TopicMessageId msgId = new TopicMessageIdImpl(topic.toString(), + (MessageIdAdv) MessageId.fromByteArray(Base64.getDecoder().decode(command.messageId))); if (log.isDebugEnabled()) { log.debug("[{}/{}] Received ack request of message {} from {} ", consumer.getTopic(), subscription, msgId, getRemote().getInetSocketAddress().toString()); From e248e14473142765db1df324c20a081a2422980e Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Fri, 21 Apr 2023 09:39:57 +0800 Subject: [PATCH 019/494] [improve][broker] Support disabling delayed bucket merging. (#20155) (cherry picked from commit e1a3fa8f6d9e3cb4e4357ee40263f55e1c1491b8) --- conf/broker.conf | 3 ++- conf/standalone.conf | 3 ++- .../java/org/apache/pulsar/broker/ServiceConfiguration.java | 5 +++-- .../broker/delayed/bucket/BucketDelayedDeliveryTracker.java | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 292b1ef1683b7..89d2d85200448 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -577,7 +577,8 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000 # The max number of delayed message index bucket, # after reaching the max buckets limitation, the adjacent buckets will be merged. -delayedDeliveryMaxNumBuckets=50 +# (disable with value -1) +delayedDeliveryMaxNumBuckets=-1 # Size of the lookahead window to use when detecting if all the messages in the topic # have a fixed delay. diff --git a/conf/standalone.conf b/conf/standalone.conf index f141946c29f4e..63bc7a29ae6da 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -1264,4 +1264,5 @@ delayedDeliveryMaxIndexesPerBucketSnapshotSegment=5000 # The max number of delayed message index bucket, # after reaching the max buckets limitation, the adjacent buckets will be merged. -delayedDeliveryMaxNumBuckets=50 +# (disable with value -1) +delayedDeliveryMaxNumBuckets=-1 diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index b31c86560f848..2c48310f96482 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -372,8 +372,9 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext(category = CATEGORY_SERVER, doc = """ The max number of delayed message index bucket, \ - after reaching the max buckets limitation, the adjacent buckets will be merged.""") - private int delayedDeliveryMaxNumBuckets = 50; + after reaching the max buckets limitation, the adjacent buckets will be merged.\ + (disable with value -1)""") + private int delayedDeliveryMaxNumBuckets = -1; @FieldContext(category = CATEGORY_SERVER, doc = "Size of the lookahead window to use " + "when detecting if all the messages in the topic have a fixed delay. " diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index 5d1dc2e27033a..b17387e276e2b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -335,7 +335,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver afterCreateImmutableBucket(immutableBucketDelayedIndexPair, createStartTime); lastMutableBucket.resetLastMutableBucketRange(); - if (immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { + if (maxNumBuckets > 0 && immutableBuckets.asMapOfRanges().size() > maxNumBuckets) { asyncMergeBucketSnapshot(); } } From 3da39b2e1553cecbc6d6b85e8bc7844f611d5637 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 21 Apr 2023 09:51:55 +0800 Subject: [PATCH 020/494] [improve][broker] Optimization protobuf code in the bucket delayed tracker (#20158) (cherry picked from commit a20d5e96acc91f8099845e8924e84cb0b5632f3f) --- pulsar-broker/pom.xml | 2 + .../BookkeeperBucketSnapshotStorage.java | 24 ++--- .../pulsar/broker/delayed/bucket/Bucket.java | 7 +- .../bucket/BucketDelayedDeliveryTracker.java | 13 ++- .../delayed/bucket/BucketSnapshotStorage.java | 4 +- .../CombinedSegmentDelayedIndexQueue.java | 22 ++++- .../delayed/bucket/DelayedIndexQueue.java | 12 ++- .../delayed/bucket/ImmutableBucket.java | 19 ++-- .../broker/delayed/bucket/MutableBucket.java | 29 +++--- .../TripleLongPriorityDelayedIndexQueue.java | 25 +++-- ...> DelayedMessageIndexBucketMetadata.proto} | 11 +-- .../DelayedMessageIndexBucketSegment.proto | 35 +++++++ .../BookkeeperBucketSnapshotStorageTest.java | 95 +++++++++---------- .../delayed/MockBucketSnapshotStorage.java | 14 +-- .../delayed/bucket/DelayedIndexQueueTest.java | 70 ++++++-------- 15 files changed, 210 insertions(+), 172 deletions(-) rename pulsar-broker/src/main/proto/{DelayedMessageIndexBucketSnapshotFormat.proto => DelayedMessageIndexBucketMetadata.proto} (85%) create mode 100644 pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 31b335e1aea68..25b31b4f2b7bd 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -566,6 +566,7 @@ **/ResourceUsage.proto **/TransactionPendingAck.proto + **/DelayedMessageIndexBucketSegment.proto @@ -610,6 +611,7 @@ ${project.basedir}/src/main/proto/TransactionPendingAck.proto ${project.basedir}/src/main/proto/ResourceUsage.proto + ${project.basedir}/src/main/proto/DelayedMessageIndexBucketSegment.proto generated-sources/lightproto/java generated-sources/lightproto/java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 18a4c322f7b27..040bbbc586f49 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import com.google.protobuf.InvalidProtocolBufferException; -import java.io.IOException; +import io.netty.buffer.ByteBuf; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -36,8 +36,8 @@ import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -126,7 +126,8 @@ private CompletableFuture addSnapshotSegments(LedgerHandle ledgerHandle, private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { try { - return SnapshotMetadata.parseFrom(ledgerEntry.getEntry()); + ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); + return SnapshotMetadata.parseFrom(entryBuffer.nioBuffer()); } catch (InvalidProtocolBufferException e) { throw new BucketSnapshotSerializationException(e); } @@ -134,15 +135,14 @@ private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { private List parseSnapshotSegmentEntries(Enumeration entryEnumeration) { List snapshotMetadataList = new ArrayList<>(); - try { - while (entryEnumeration.hasMoreElements()) { - LedgerEntry ledgerEntry = entryEnumeration.nextElement(); - snapshotMetadataList.add(SnapshotSegment.parseFrom(ledgerEntry.getEntry())); - } - return snapshotMetadataList; - } catch (IOException e) { - throw new BucketSnapshotSerializationException(e); + while (entryEnumeration.hasMoreElements()) { + LedgerEntry ledgerEntry = entryEnumeration.nextElement(); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); + snapshotSegment.parseFrom(entryBuffer, entryBuffer.readableBytes()); + snapshotMetadataList.add(snapshotSegment); } + return snapshotMetadataList; } @NotNull diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 4d7d3aa512be6..a1693b1553d97 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -31,7 +31,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.RoaringBitmap; @@ -132,8 +133,8 @@ long getAndUpdateBucketId() { } CompletableFuture asyncSaveBucketSnapshot( - ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata, - List bucketSnapshotSegments) { + ImmutableBucket bucket, SnapshotMetadata snapshotMetadata, + List bucketSnapshotSegments) { final String bucketKey = bucket.bucketKey(); final String cursorName = Codec.decode(cursor.getName()); final String topicName = dispatcherName.substring(0, dispatcherName.lastIndexOf(" / " + cursorName)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index b17387e276e2b..c90064c9137bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -53,8 +53,8 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.util.FutureUtil; @@ -286,8 +286,7 @@ private void afterCreateImmutableBucket(Pair immu // Put indexes back into the shared queue and downgrade to memory mode synchronized (BucketDelayedDeliveryTracker.this) { immutableBucket.getSnapshotSegments().ifPresent(snapshotSegments -> { - for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment : - snapshotSegments) { + for (SnapshotSegment snapshotSegment : snapshotSegments) { for (DelayedIndex delayedIndex : snapshotSegment.getIndexesList()) { sharedBucketPriorityQueue.add(delayedIndex.getTimestamp(), delayedIndex.getLedgerId(), delayedIndex.getEntryId()); @@ -450,7 +449,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List>> getRemainFutures = + List>> getRemainFutures = buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList(); return FutureUtil.waitForAll(getRemainFutures) @@ -601,11 +600,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa bucket.asyncDeleteBucketSnapshot(stats); return; } - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex + DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId(), bucket); - for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : indexList) { + for (DelayedIndex index : indexList) { sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), index.getEntryId()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java index 51c89bed47af2..7464ef9cd3f63 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java @@ -20,8 +20,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; public interface BucketSnapshotStorage { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java index 5655a26878296..006938e9ed271 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java @@ -24,8 +24,8 @@ import java.util.PriorityQueue; import javax.annotation.concurrent.NotThreadSafe; import lombok.AllArgsConstructor; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; @NotThreadSafe class CombinedSegmentDelayedIndexQueue implements DelayedIndexQueue { @@ -40,8 +40,8 @@ static class Node { } private static final Comparator COMPARATOR_NODE = (node1, node2) -> DelayedIndexQueue.COMPARATOR.compare( - node1.segmentList.get(node1.segmentListCursor).getIndexes(node1.segmentCursor), - node2.segmentList.get(node2.segmentListCursor).getIndexes(node2.segmentCursor)); + node1.segmentList.get(node1.segmentListCursor).getIndexeAt(node1.segmentCursor), + node2.segmentList.get(node2.segmentListCursor).getIndexeAt(node2.segmentCursor)); private final PriorityQueue kpq; @@ -77,7 +77,7 @@ private DelayedIndex getValue(boolean needAdvanceCursor) { Objects.requireNonNull(node); SnapshotSegment snapshotSegment = node.segmentList.get(node.segmentListCursor); - DelayedIndex delayedIndex = snapshotSegment.getIndexes(node.segmentCursor); + DelayedIndex delayedIndex = snapshotSegment.getIndexeAt(node.segmentCursor); if (!needAdvanceCursor) { return delayedIndex; } @@ -104,4 +104,16 @@ private DelayedIndex getValue(boolean needAdvanceCursor) { return delayedIndex; } + + @Override + public void popToObject(DelayedIndex delayedIndex) { + DelayedIndex value = getValue(true); + delayedIndex.copyFrom(value); + } + + @Override + public long peekTimestamp() { + DelayedIndex value = getValue(false); + return value.getTimestamp(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java index dee476c376ec9..f1209a3137a45 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java @@ -20,10 +20,10 @@ import java.util.Comparator; import java.util.Objects; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; interface DelayedIndexQueue { - Comparator COMPARATOR = (o1, o2) -> { + Comparator COMPARATOR = (o1, o2) -> { if (!Objects.equals(o1.getTimestamp(), o2.getTimestamp())) { return Long.compare(o1.getTimestamp(), o2.getTimestamp()); } else if (!Objects.equals(o1.getLedgerId(), o2.getLedgerId())) { @@ -35,7 +35,11 @@ interface DelayedIndexQueue { boolean isEmpty(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek(); + DelayedIndex peek(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop(); + DelayedIndex pop(); + + void popToObject(DelayedIndex delayedIndex); + + long peekTimestamp(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 57de5c84fcd82..0932f51f350ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -32,9 +32,9 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.mutable.MutableLong; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.InvalidRoaringFormat; import org.roaringbitmap.RoaringBitmap; @@ -43,7 +43,7 @@ class ImmutableBucket extends Bucket { @Setter - private List snapshotSegments; + private List snapshotSegments; boolean merging = false; @@ -55,7 +55,7 @@ class ImmutableBucket extends Bucket { super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId); } - public Optional> getSnapshotSegments() { + public Optional> getSnapshotSegments() { return Optional.ofNullable(snapshotSegments); } @@ -84,7 +84,7 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b } }), BucketSnapshotPersistenceException.class, MaxRetryTimes) .thenApply(snapshotMetadata -> { - List metadataList = + List metadataList = snapshotMetadata.getMetadataListList(); // Skip all already reach schedule time snapshot segments @@ -125,10 +125,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b return Collections.emptyList(); } - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - List indexList = - snapshotSegment.getIndexesList(); + List indexList = snapshotSegment.getIndexesList(); this.setCurrentSegmentEntryId(nextSegmentEntryId); if (isRecover) { this.asyncUpdateSnapshotLength(); @@ -171,7 +170,7 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, setNumberBucketDelayedMessages(numberMessages.getValue()); } - CompletableFuture> getRemainSnapshotSegment() { + CompletableFuture> getRemainSnapshotSegment() { int nextSegmentEntryId = currentSegmentEntryId + 1; if (nextSegmentEntryId > lastSegmentEntryId) { return CompletableFuture.completedFuture(Collections.emptyList()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index f404d5d02c15a..b7e9e68f1bdc7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -30,10 +30,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.roaringbitmap.RoaringBitmap; @@ -75,14 +75,16 @@ Pair createImmutableBucketAndAsyncPersistent( List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); Map bitMap = new HashMap<>(); - SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); + SnapshotSegment snapshotSegment = new SnapshotSegment(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); List firstScheduleTimestamps = new ArrayList<>(); long currentTimestampUpperLimit = 0; long currentFirstTimestamp = 0L; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex delayedIndex = delayedIndexQueue.peek(); + DelayedIndex delayedIndex = snapshotSegment.addIndexe(); + delayedIndexQueue.popToObject(delayedIndex); + long timestamp = delayedIndex.getTimestamp(); if (currentTimestampUpperLimit == 0) { currentFirstTimestamp = timestamp; @@ -100,16 +102,13 @@ Pair createImmutableBucketAndAsyncPersistent( sharedQueue.add(timestamp, ledgerId, entryId); } - delayedIndexQueue.pop(); numMessages++; bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); - snapshotSegmentBuilder.addIndexes(delayedIndex); - - if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit + if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peekTimestamp() > currentTimestampUpperLimit || (maxIndexesPerBucketSnapshotSegment != -1 - && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { + && snapshotSegment.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp); currentTimestampUpperLimit = 0; @@ -129,8 +128,8 @@ Pair createImmutableBucketAndAsyncPersistent( segmentMetadataList.add(segmentMetadataBuilder.build()); segmentMetadataBuilder.clear(); - bucketSnapshotSegments.add(snapshotSegmentBuilder.build()); - snapshotSegmentBuilder.clear(); + bucketSnapshotSegments.add(snapshotSegment); + snapshotSegment = new SnapshotSegment(); } } @@ -153,8 +152,8 @@ Pair createImmutableBucketAndAsyncPersistent( // Add the first snapshot segment last message to snapshotSegmentLastMessageTable checkArgument(!bucketSnapshotSegments.isEmpty()); - SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - DelayedIndex lastDelayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1); + SnapshotSegment firstSnapshotSegment = bucketSnapshotSegments.get(0); + DelayedIndex lastDelayedIndex = firstSnapshotSegment.getIndexeAt(firstSnapshotSegment.getIndexesCount() - 1); Pair result = Pair.of(bucket, lastDelayedIndex); CompletableFuture future = asyncSaveBucketSnapshot(bucket, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java index b8d54bd78b428..4faee3b17f17f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import javax.annotation.concurrent.NotThreadSafe; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; @NotThreadSafe @@ -41,17 +41,28 @@ public boolean isEmpty() { } @Override - public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek() { - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setTimestamp(queue.peekN1()) - .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()).build(); + public DelayedIndex peek() { + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(queue.peekN1()) + .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()); return delayedIndex; } @Override - public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop() { - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek = peek(); + public DelayedIndex pop() { + DelayedIndex peek = peek(); queue.pop(); return peek; } + + @Override + public void popToObject(DelayedIndex delayedIndex) { + delayedIndex.setTimestamp(queue.peekN1()) + .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()); + queue.pop(); + } + + @Override + public long peekTimestamp() { + return queue.peekN1(); + } } diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto similarity index 85% rename from pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto rename to pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto index 6996b860c5249..01b770c567d09 100644 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto @@ -21,12 +21,7 @@ syntax = "proto2"; package pulsar.delay; option java_package = "org.apache.pulsar.broker.delayed.proto"; option optimize_for = SPEED; - -message DelayedIndex { - required uint64 timestamp = 1; - required uint64 ledger_id = 2; - required uint64 entry_id = 3; -} +option java_multiple_files = true; message SnapshotSegmentMetadata { map delayed_index_bit_map = 1; @@ -34,10 +29,6 @@ message SnapshotSegmentMetadata { required uint64 min_schedule_timestamp = 3; } -message SnapshotSegment { - repeated DelayedIndex indexes = 1; -} - message SnapshotMetadata { repeated SnapshotSegmentMetadata metadata_list = 1; } diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto new file mode 100644 index 0000000000000..633d6a8f1615c --- /dev/null +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +syntax = "proto2"; + +package pulsar.delay; +option java_package = "org.apache.pulsar.broker.delayed.proto"; +option optimize_for = SPEED; +option java_multiple_files = true; + +message DelayedIndex { + required uint64 timestamp = 1; + required uint64 ledger_id = 2; + required uint64 entry_id = 3; +} + +message SnapshotSegment { + repeated DelayedIndex indexes = 1; + map delayed_index_bit_map = 2; +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index 7cb6b8d5865bb..d26f38fa2bc2b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -29,7 +29,10 @@ import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -60,9 +63,8 @@ protected void cleanup() throws Exception { @Test public void testCreateSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build(); - List bucketSnapshotSegments = new ArrayList<>(); + SnapshotMetadata snapshotMetadata = SnapshotMetadata.newBuilder().build(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); @@ -72,24 +74,23 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException @Test public void testGetSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -99,13 +100,13 @@ public void testGetSnapshot() throws ExecutionException, InterruptedException { Long bucketId = future.get(); Assert.assertNotNull(bucketId); - CompletableFuture> bucketSnapshotSegment = + CompletableFuture> bucketSnapshotSegment = bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3); - List snapshotSegments = bucketSnapshotSegment.get(); + List snapshotSegments = bucketSnapshotSegment.get(); Assert.assertEquals(2, snapshotSegments.size()); - for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment segment : snapshotSegments) { - for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : segment.getIndexesList()) { + for (SnapshotSegment segment : snapshotSegments) { + for (DelayedIndex index : segment.getIndexesList()) { Assert.assertEquals(100L, index.getLedgerId()); Assert.assertEquals(10L, index.getEntryId()); Assert.assertEquals(timeMillis, index.getTimestamp()); @@ -121,17 +122,17 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce map.put(100L, ByteString.copyFrom("test1", StandardCharsets.UTF_8)); map.put(200L, ByteString.copyFrom("test2", StandardCharsets.UTF_8)); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMaxScheduleTimestamp(timeMillis) .setMinScheduleTimestamp(timeMillis) .putAllDelayedIndexBitMap(map).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, @@ -139,10 +140,10 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce Long bucketId = future.get(); Assert.assertNotNull(bucketId); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata bucketSnapshotMetadata = + SnapshotMetadata bucketSnapshotMetadata = bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId).get(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata metadata = + SnapshotSegmentMetadata metadata = bucketSnapshotMetadata.getMetadataList(0); Assert.assertEquals(timeMillis, metadata.getMaxScheduleTimestamp()); @@ -152,9 +153,9 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce @Test public void testDeleteSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build(); - List bucketSnapshotSegments = new ArrayList<>(); + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder().build(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); @@ -173,24 +174,22 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException @Test public void testGetBucketSnapshotLength() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -207,24 +206,22 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted @Test public void testConcurrencyGet() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -237,7 +234,7 @@ public void testConcurrencyGet() throws ExecutionException, InterruptedException List> futures = new ArrayList<>(); for (int i = 0; i < 100; i++) { CompletableFuture future0 = CompletableFuture.runAsync(() -> { - List list = + List list = bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join(); Assert.assertTrue(list.size() > 0); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index 9e924bdeda341..dc1c0e09ca276 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -36,8 +36,8 @@ import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -139,13 +139,9 @@ public CompletableFuture> getBucketSnapshotSegment(long bu long lastEntryId = Math.min(lastSegmentEntryId, this.bucketSnapshots.get(bucketId).size()); for (int i = (int) firstSegmentEntryId; i <= lastEntryId ; i++) { ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(i); - SnapshotSegment snapshotSegment; - try { - snapshotSegment = SnapshotSegment.parseFrom(byteBuf.nioBuffer()); - snapshotSegments.add(snapshotSegment); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.parseFrom(byteBuf, byteBuf.readableBytes()); + snapshotSegments.add(snapshotSegment); } return snapshotSegments; }, executorService); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java index 8f87f0d49a20c..5dc3dcc7cb9a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java @@ -23,8 +23,8 @@ import java.util.List; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.testng.Assert; import org.testng.annotations.Test; @@ -35,27 +35,19 @@ public class DelayedIndexQueueTest { @Test public void testCompare() { DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(2).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(2).setLedgerId(2L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(2L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); - delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); - delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(2L) - .build(); + delayedIndex = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); + delayedIndex2 = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); } @@ -63,21 +55,19 @@ public void testCompare() { public void testCombinedSegmentDelayedIndexQueue() { List listA = new ArrayList<>(); for (int i = 0; i < 10; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L); listA.add(delayedIndex); } - SnapshotSegment snapshotSegmentA1 = SnapshotSegment.newBuilder().addAllIndexes(listA).build(); + SnapshotSegment snapshotSegmentA1 = new SnapshotSegment(); + snapshotSegmentA1.addAllIndexes(listA); List listA2 = new ArrayList<>(); for (int i = 10; i < 20; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L); listA2.add(delayedIndex); } - SnapshotSegment snapshotSegmentA2 = SnapshotSegment.newBuilder().addAllIndexes(listA2).build(); + SnapshotSegment snapshotSegmentA2 = new SnapshotSegment(); + snapshotSegmentA2.addAllIndexes(listA2); List segmentListA = new ArrayList<>(); segmentListA.add(snapshotSegmentA1); @@ -85,36 +75,32 @@ public void testCombinedSegmentDelayedIndexQueue() { List listB = new ArrayList<>(); for (int i = 0; i < 9; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L); - DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) - .build(); + DelayedIndex delayedIndex2 = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L); listB.add(delayedIndex); listB.add(delayedIndex2); } - SnapshotSegment snapshotSegmentB = SnapshotSegment.newBuilder().addAllIndexes(listB).build(); + SnapshotSegment snapshotSegmentB = new SnapshotSegment(); + snapshotSegmentB.addAllIndexes(listB); List segmentListB = new ArrayList<>(); segmentListB.add(snapshotSegmentB); - segmentListB.add(SnapshotSegment.newBuilder().build()); + segmentListB.add(new SnapshotSegment()); List listC = new ArrayList<>(); for (int i = 10; i < 30; i+=2) { DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L); DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L); listC.add(delayedIndex); listC.add(delayedIndex2); } - SnapshotSegment snapshotSegmentC = SnapshotSegment.newBuilder().addAllIndexes(listC).build(); + SnapshotSegment snapshotSegmentC = new SnapshotSegment(); + snapshotSegmentC.addAllIndexes(listC); List segmentListC = new ArrayList<>(); segmentListC.add(snapshotSegmentC); @@ -123,11 +109,14 @@ public void testCombinedSegmentDelayedIndexQueue() { int count = 0; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex pop = delayedIndexQueue.pop(); + DelayedIndex pop = new DelayedIndex(); + delayedIndexQueue.popToObject(pop); log.info("{} , {}, {}", pop.getTimestamp(), pop.getLedgerId(), pop.getEntryId()); count++; if (!delayedIndexQueue.isEmpty()) { DelayedIndex peek = delayedIndexQueue.peek(); + long timestamp = delayedIndexQueue.peekTimestamp(); + Assert.assertEquals(timestamp, peek.getTimestamp()); Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } @@ -147,10 +136,13 @@ public void TripleLongPriorityDelayedIndexQueueTest() { int count = 0; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex pop = delayedIndexQueue.pop(); + DelayedIndex pop = new DelayedIndex(); + delayedIndexQueue.popToObject(pop); count++; if (!delayedIndexQueue.isEmpty()) { DelayedIndex peek = delayedIndexQueue.peek(); + long timestamp = delayedIndexQueue.peekTimestamp(); + Assert.assertEquals(timestamp, peek.getTimestamp()); Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } From e3e1c08e513823bba2758da70f32079a3aa0d318 Mon Sep 17 00:00:00 2001 From: coderzc Date: Sun, 23 Apr 2023 23:53:43 +0800 Subject: [PATCH 021/494] Revert "[improve][broker] Optimization protobuf code in the bucket delayed tracker (#20158)" This reverts commit 3da39b2e1553cecbc6d6b85e8bc7844f611d5637. --- pulsar-broker/pom.xml | 2 - .../BookkeeperBucketSnapshotStorage.java | 24 ++--- .../pulsar/broker/delayed/bucket/Bucket.java | 7 +- .../bucket/BucketDelayedDeliveryTracker.java | 13 +-- .../delayed/bucket/BucketSnapshotStorage.java | 4 +- .../CombinedSegmentDelayedIndexQueue.java | 22 +---- .../delayed/bucket/DelayedIndexQueue.java | 12 +-- .../delayed/bucket/ImmutableBucket.java | 19 ++-- .../broker/delayed/bucket/MutableBucket.java | 29 +++--- .../TripleLongPriorityDelayedIndexQueue.java | 25 ++--- .../DelayedMessageIndexBucketSegment.proto | 35 ------- ...yedMessageIndexBucketSnapshotFormat.proto} | 11 ++- .../BookkeeperBucketSnapshotStorageTest.java | 95 ++++++++++--------- .../delayed/MockBucketSnapshotStorage.java | 14 ++- .../delayed/bucket/DelayedIndexQueueTest.java | 70 ++++++++------ 15 files changed, 172 insertions(+), 210 deletions(-) delete mode 100644 pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto rename pulsar-broker/src/main/proto/{DelayedMessageIndexBucketMetadata.proto => DelayedMessageIndexBucketSnapshotFormat.proto} (85%) diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 25b31b4f2b7bd..31b335e1aea68 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -566,7 +566,6 @@ **/ResourceUsage.proto **/TransactionPendingAck.proto - **/DelayedMessageIndexBucketSegment.proto @@ -611,7 +610,6 @@ ${project.basedir}/src/main/proto/TransactionPendingAck.proto ${project.basedir}/src/main/proto/ResourceUsage.proto - ${project.basedir}/src/main/proto/DelayedMessageIndexBucketSegment.proto generated-sources/lightproto/java generated-sources/lightproto/java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 040bbbc586f49..18a4c322f7b27 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import com.google.protobuf.InvalidProtocolBufferException; -import io.netty.buffer.ByteBuf; +import java.io.IOException; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -36,8 +36,8 @@ import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -126,8 +126,7 @@ private CompletableFuture addSnapshotSegments(LedgerHandle ledgerHandle, private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { try { - ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); - return SnapshotMetadata.parseFrom(entryBuffer.nioBuffer()); + return SnapshotMetadata.parseFrom(ledgerEntry.getEntry()); } catch (InvalidProtocolBufferException e) { throw new BucketSnapshotSerializationException(e); } @@ -135,14 +134,15 @@ private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { private List parseSnapshotSegmentEntries(Enumeration entryEnumeration) { List snapshotMetadataList = new ArrayList<>(); - while (entryEnumeration.hasMoreElements()) { - LedgerEntry ledgerEntry = entryEnumeration.nextElement(); - SnapshotSegment snapshotSegment = new SnapshotSegment(); - ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); - snapshotSegment.parseFrom(entryBuffer, entryBuffer.readableBytes()); - snapshotMetadataList.add(snapshotSegment); + try { + while (entryEnumeration.hasMoreElements()) { + LedgerEntry ledgerEntry = entryEnumeration.nextElement(); + snapshotMetadataList.add(SnapshotSegment.parseFrom(ledgerEntry.getEntry())); + } + return snapshotMetadataList; + } catch (IOException e) { + throw new BucketSnapshotSerializationException(e); } - return snapshotMetadataList; } @NotNull diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index a1693b1553d97..4d7d3aa512be6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -31,8 +31,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.RoaringBitmap; @@ -133,8 +132,8 @@ long getAndUpdateBucketId() { } CompletableFuture asyncSaveBucketSnapshot( - ImmutableBucket bucket, SnapshotMetadata snapshotMetadata, - List bucketSnapshotSegments) { + ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata, + List bucketSnapshotSegments) { final String bucketKey = bucket.bucketKey(); final String cursorName = Codec.decode(cursor.getName()); final String topicName = dispatcherName.substring(0, dispatcherName.lastIndexOf(" / " + cursorName)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index c90064c9137bb..b17387e276e2b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -53,8 +53,8 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; -import org.apache.pulsar.broker.delayed.proto.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.util.FutureUtil; @@ -286,7 +286,8 @@ private void afterCreateImmutableBucket(Pair immu // Put indexes back into the shared queue and downgrade to memory mode synchronized (BucketDelayedDeliveryTracker.this) { immutableBucket.getSnapshotSegments().ifPresent(snapshotSegments -> { - for (SnapshotSegment snapshotSegment : snapshotSegments) { + for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment : + snapshotSegments) { for (DelayedIndex delayedIndex : snapshotSegment.getIndexesList()) { sharedBucketPriorityQueue.add(delayedIndex.getTimestamp(), delayedIndex.getLedgerId(), delayedIndex.getEntryId()); @@ -449,7 +450,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List>> getRemainFutures = + List>> getRemainFutures = buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList(); return FutureUtil.waitForAll(getRemainFutures) @@ -600,11 +601,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa bucket.asyncDeleteBucketSnapshot(stats); return; } - DelayedIndex + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId(), bucket); - for (DelayedIndex index : indexList) { + for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : indexList) { sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), index.getEntryId()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java index 7464ef9cd3f63..51c89bed47af2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java @@ -20,8 +20,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; public interface BucketSnapshotStorage { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java index 006938e9ed271..5655a26878296 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java @@ -24,8 +24,8 @@ import java.util.PriorityQueue; import javax.annotation.concurrent.NotThreadSafe; import lombok.AllArgsConstructor; -import org.apache.pulsar.broker.delayed.proto.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; @NotThreadSafe class CombinedSegmentDelayedIndexQueue implements DelayedIndexQueue { @@ -40,8 +40,8 @@ static class Node { } private static final Comparator COMPARATOR_NODE = (node1, node2) -> DelayedIndexQueue.COMPARATOR.compare( - node1.segmentList.get(node1.segmentListCursor).getIndexeAt(node1.segmentCursor), - node2.segmentList.get(node2.segmentListCursor).getIndexeAt(node2.segmentCursor)); + node1.segmentList.get(node1.segmentListCursor).getIndexes(node1.segmentCursor), + node2.segmentList.get(node2.segmentListCursor).getIndexes(node2.segmentCursor)); private final PriorityQueue kpq; @@ -77,7 +77,7 @@ private DelayedIndex getValue(boolean needAdvanceCursor) { Objects.requireNonNull(node); SnapshotSegment snapshotSegment = node.segmentList.get(node.segmentListCursor); - DelayedIndex delayedIndex = snapshotSegment.getIndexeAt(node.segmentCursor); + DelayedIndex delayedIndex = snapshotSegment.getIndexes(node.segmentCursor); if (!needAdvanceCursor) { return delayedIndex; } @@ -104,16 +104,4 @@ private DelayedIndex getValue(boolean needAdvanceCursor) { return delayedIndex; } - - @Override - public void popToObject(DelayedIndex delayedIndex) { - DelayedIndex value = getValue(true); - delayedIndex.copyFrom(value); - } - - @Override - public long peekTimestamp() { - DelayedIndex value = getValue(false); - return value.getTimestamp(); - } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java index f1209a3137a45..dee476c376ec9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java @@ -20,10 +20,10 @@ import java.util.Comparator; import java.util.Objects; -import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; interface DelayedIndexQueue { - Comparator COMPARATOR = (o1, o2) -> { + Comparator COMPARATOR = (o1, o2) -> { if (!Objects.equals(o1.getTimestamp(), o2.getTimestamp())) { return Long.compare(o1.getTimestamp(), o2.getTimestamp()); } else if (!Objects.equals(o1.getLedgerId(), o2.getLedgerId())) { @@ -35,11 +35,7 @@ interface DelayedIndexQueue { boolean isEmpty(); - DelayedIndex peek(); + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek(); - DelayedIndex pop(); - - void popToObject(DelayedIndex delayedIndex); - - long peekTimestamp(); + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 0932f51f350ce..57de5c84fcd82 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -32,9 +32,9 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.mutable.MutableLong; -import org.apache.pulsar.broker.delayed.proto.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.InvalidRoaringFormat; import org.roaringbitmap.RoaringBitmap; @@ -43,7 +43,7 @@ class ImmutableBucket extends Bucket { @Setter - private List snapshotSegments; + private List snapshotSegments; boolean merging = false; @@ -55,7 +55,7 @@ class ImmutableBucket extends Bucket { super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId); } - public Optional> getSnapshotSegments() { + public Optional> getSnapshotSegments() { return Optional.ofNullable(snapshotSegments); } @@ -84,7 +84,7 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b } }), BucketSnapshotPersistenceException.class, MaxRetryTimes) .thenApply(snapshotMetadata -> { - List metadataList = + List metadataList = snapshotMetadata.getMetadataListList(); // Skip all already reach schedule time snapshot segments @@ -125,9 +125,10 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b return Collections.emptyList(); } - SnapshotSegment snapshotSegment = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - List indexList = snapshotSegment.getIndexesList(); + List indexList = + snapshotSegment.getIndexesList(); this.setCurrentSegmentEntryId(nextSegmentEntryId); if (isRecover) { this.asyncUpdateSnapshotLength(); @@ -170,7 +171,7 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, setNumberBucketDelayedMessages(numberMessages.getValue()); } - CompletableFuture> getRemainSnapshotSegment() { + CompletableFuture> getRemainSnapshotSegment() { int nextSegmentEntryId = currentSegmentEntryId + 1; if (nextSegmentEntryId > lastSegmentEntryId) { return CompletableFuture.completedFuture(Collections.emptyList()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index b7e9e68f1bdc7..f404d5d02c15a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -30,10 +30,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.delayed.proto.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.roaringbitmap.RoaringBitmap; @@ -75,16 +75,14 @@ Pair createImmutableBucketAndAsyncPersistent( List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); Map bitMap = new HashMap<>(); - SnapshotSegment snapshotSegment = new SnapshotSegment(); + SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); List firstScheduleTimestamps = new ArrayList<>(); long currentTimestampUpperLimit = 0; long currentFirstTimestamp = 0L; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex delayedIndex = snapshotSegment.addIndexe(); - delayedIndexQueue.popToObject(delayedIndex); - + DelayedIndex delayedIndex = delayedIndexQueue.peek(); long timestamp = delayedIndex.getTimestamp(); if (currentTimestampUpperLimit == 0) { currentFirstTimestamp = timestamp; @@ -102,13 +100,16 @@ Pair createImmutableBucketAndAsyncPersistent( sharedQueue.add(timestamp, ledgerId, entryId); } + delayedIndexQueue.pop(); numMessages++; bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); - if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peekTimestamp() > currentTimestampUpperLimit + snapshotSegmentBuilder.addIndexes(delayedIndex); + + if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit || (maxIndexesPerBucketSnapshotSegment != -1 - && snapshotSegment.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { + && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp); currentTimestampUpperLimit = 0; @@ -128,8 +129,8 @@ Pair createImmutableBucketAndAsyncPersistent( segmentMetadataList.add(segmentMetadataBuilder.build()); segmentMetadataBuilder.clear(); - bucketSnapshotSegments.add(snapshotSegment); - snapshotSegment = new SnapshotSegment(); + bucketSnapshotSegments.add(snapshotSegmentBuilder.build()); + snapshotSegmentBuilder.clear(); } } @@ -152,8 +153,8 @@ Pair createImmutableBucketAndAsyncPersistent( // Add the first snapshot segment last message to snapshotSegmentLastMessageTable checkArgument(!bucketSnapshotSegments.isEmpty()); - SnapshotSegment firstSnapshotSegment = bucketSnapshotSegments.get(0); - DelayedIndex lastDelayedIndex = firstSnapshotSegment.getIndexeAt(firstSnapshotSegment.getIndexesCount() - 1); + SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); + DelayedIndex lastDelayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1); Pair result = Pair.of(bucket, lastDelayedIndex); CompletableFuture future = asyncSaveBucketSnapshot(bucket, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java index 4faee3b17f17f..b8d54bd78b428 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import javax.annotation.concurrent.NotThreadSafe; -import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; @NotThreadSafe @@ -41,28 +41,17 @@ public boolean isEmpty() { } @Override - public DelayedIndex peek() { - DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(queue.peekN1()) - .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()); + public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek() { + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setTimestamp(queue.peekN1()) + .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()).build(); return delayedIndex; } @Override - public DelayedIndex pop() { - DelayedIndex peek = peek(); + public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop() { + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek = peek(); queue.pop(); return peek; } - - @Override - public void popToObject(DelayedIndex delayedIndex) { - delayedIndex.setTimestamp(queue.peekN1()) - .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()); - queue.pop(); - } - - @Override - public long peekTimestamp() { - return queue.peekN1(); - } } diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto deleted file mode 100644 index 633d6a8f1615c..0000000000000 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -syntax = "proto2"; - -package pulsar.delay; -option java_package = "org.apache.pulsar.broker.delayed.proto"; -option optimize_for = SPEED; -option java_multiple_files = true; - -message DelayedIndex { - required uint64 timestamp = 1; - required uint64 ledger_id = 2; - required uint64 entry_id = 3; -} - -message SnapshotSegment { - repeated DelayedIndex indexes = 1; - map delayed_index_bit_map = 2; -} diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto similarity index 85% rename from pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto rename to pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto index 01b770c567d09..6996b860c5249 100644 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto @@ -21,7 +21,12 @@ syntax = "proto2"; package pulsar.delay; option java_package = "org.apache.pulsar.broker.delayed.proto"; option optimize_for = SPEED; -option java_multiple_files = true; + +message DelayedIndex { + required uint64 timestamp = 1; + required uint64 ledger_id = 2; + required uint64 entry_id = 3; +} message SnapshotSegmentMetadata { map delayed_index_bit_map = 1; @@ -29,6 +34,10 @@ message SnapshotSegmentMetadata { required uint64 min_schedule_timestamp = 3; } +message SnapshotSegment { + repeated DelayedIndex indexes = 1; +} + message SnapshotMetadata { repeated SnapshotSegmentMetadata metadata_list = 1; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index d26f38fa2bc2b..7cb6b8d5865bb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -29,10 +29,7 @@ import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; -import org.apache.pulsar.broker.delayed.proto.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -63,8 +60,9 @@ protected void cleanup() throws Exception { @Test public void testCreateSnapshot() throws ExecutionException, InterruptedException { - SnapshotMetadata snapshotMetadata = SnapshotMetadata.newBuilder().build(); - List bucketSnapshotSegments = new ArrayList<>(); + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); @@ -74,23 +72,24 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException @Test public void testGetSnapshot() throws ExecutionException, InterruptedException { - SnapshotSegmentMetadata segmentMetadata = - SnapshotSegmentMetadata.newBuilder() + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - SnapshotMetadata snapshotMetadata = - SnapshotMetadata.newBuilder() + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis); - SnapshotSegment snapshotSegment = new SnapshotSegment(); - snapshotSegment.addIndexe().copyFrom(delayedIndex); + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis).build(); + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -100,13 +99,13 @@ public void testGetSnapshot() throws ExecutionException, InterruptedException { Long bucketId = future.get(); Assert.assertNotNull(bucketId); - CompletableFuture> bucketSnapshotSegment = + CompletableFuture> bucketSnapshotSegment = bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3); - List snapshotSegments = bucketSnapshotSegment.get(); + List snapshotSegments = bucketSnapshotSegment.get(); Assert.assertEquals(2, snapshotSegments.size()); - for (SnapshotSegment segment : snapshotSegments) { - for (DelayedIndex index : segment.getIndexesList()) { + for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment segment : snapshotSegments) { + for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : segment.getIndexesList()) { Assert.assertEquals(100L, index.getLedgerId()); Assert.assertEquals(10L, index.getEntryId()); Assert.assertEquals(timeMillis, index.getTimestamp()); @@ -122,17 +121,17 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce map.put(100L, ByteString.copyFrom("test1", StandardCharsets.UTF_8)); map.put(200L, ByteString.copyFrom("test2", StandardCharsets.UTF_8)); - SnapshotSegmentMetadata segmentMetadata = - SnapshotSegmentMetadata.newBuilder() + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() .setMaxScheduleTimestamp(timeMillis) .setMinScheduleTimestamp(timeMillis) .putAllDelayedIndexBitMap(map).build(); - SnapshotMetadata snapshotMetadata = - SnapshotMetadata.newBuilder() + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, @@ -140,10 +139,10 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce Long bucketId = future.get(); Assert.assertNotNull(bucketId); - SnapshotMetadata bucketSnapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata bucketSnapshotMetadata = bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId).get(); - SnapshotSegmentMetadata metadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata metadata = bucketSnapshotMetadata.getMetadataList(0); Assert.assertEquals(timeMillis, metadata.getMaxScheduleTimestamp()); @@ -153,9 +152,9 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce @Test public void testDeleteSnapshot() throws ExecutionException, InterruptedException { - SnapshotMetadata snapshotMetadata = - SnapshotMetadata.newBuilder().build(); - List bucketSnapshotSegments = new ArrayList<>(); + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); @@ -174,22 +173,24 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException @Test public void testGetBucketSnapshotLength() throws ExecutionException, InterruptedException { - SnapshotSegmentMetadata segmentMetadata = - SnapshotSegmentMetadata.newBuilder() + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - SnapshotMetadata snapshotMetadata = - SnapshotMetadata.newBuilder() + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis); - SnapshotSegment snapshotSegment = new SnapshotSegment(); - snapshotSegment.addIndexe().copyFrom(delayedIndex); + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis).build(); + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -206,22 +207,24 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted @Test public void testConcurrencyGet() throws ExecutionException, InterruptedException { - SnapshotSegmentMetadata segmentMetadata = - SnapshotSegmentMetadata.newBuilder() + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - SnapshotMetadata snapshotMetadata = - SnapshotMetadata.newBuilder() + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis); - SnapshotSegment snapshotSegment = new SnapshotSegment(); - snapshotSegment.addIndexe().copyFrom(delayedIndex); + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis).build(); + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -234,7 +237,7 @@ public void testConcurrencyGet() throws ExecutionException, InterruptedException List> futures = new ArrayList<>(); for (int i = 0; i < 100; i++) { CompletableFuture future0 = CompletableFuture.runAsync(() -> { - List list = + List list = bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join(); Assert.assertTrue(list.size() > 0); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index dc1c0e09ca276..9e924bdeda341 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -36,8 +36,8 @@ import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; -import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -139,9 +139,13 @@ public CompletableFuture> getBucketSnapshotSegment(long bu long lastEntryId = Math.min(lastSegmentEntryId, this.bucketSnapshots.get(bucketId).size()); for (int i = (int) firstSegmentEntryId; i <= lastEntryId ; i++) { ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(i); - SnapshotSegment snapshotSegment = new SnapshotSegment(); - snapshotSegment.parseFrom(byteBuf, byteBuf.readableBytes()); - snapshotSegments.add(snapshotSegment); + SnapshotSegment snapshotSegment; + try { + snapshotSegment = SnapshotSegment.parseFrom(byteBuf.nioBuffer()); + snapshotSegments.add(snapshotSegment); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } } return snapshotSegments; }, executorService); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java index 5dc3dcc7cb9a7..8f87f0d49a20c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java @@ -23,8 +23,8 @@ import java.util.List; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.delayed.proto.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.testng.Assert; import org.testng.annotations.Test; @@ -35,19 +35,27 @@ public class DelayedIndexQueueTest { @Test public void testCompare() { DelayedIndex delayedIndex = - new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); + DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) + .build(); DelayedIndex delayedIndex2 = - new DelayedIndex().setTimestamp(2).setLedgerId(2L).setEntryId(2L); + DelayedIndex.newBuilder().setTimestamp(2).setLedgerId(2L).setEntryId(2L) + .build(); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); delayedIndex = - new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); + DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) + .build(); delayedIndex2 = - new DelayedIndex().setTimestamp(1).setLedgerId(2L).setEntryId(2L); + DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(2L).setEntryId(2L) + .build(); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); - delayedIndex = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); - delayedIndex2 = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(2L); + delayedIndex = + DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) + .build(); + delayedIndex2 = + DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(2L) + .build(); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); } @@ -55,19 +63,21 @@ public void testCompare() { public void testCombinedSegmentDelayedIndexQueue() { List listA = new ArrayList<>(); for (int i = 0; i < 10; i++) { - DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L); + DelayedIndex delayedIndex = + DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L) + .build(); listA.add(delayedIndex); } - SnapshotSegment snapshotSegmentA1 = new SnapshotSegment(); - snapshotSegmentA1.addAllIndexes(listA); + SnapshotSegment snapshotSegmentA1 = SnapshotSegment.newBuilder().addAllIndexes(listA).build(); List listA2 = new ArrayList<>(); for (int i = 10; i < 20; i++) { - DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L); + DelayedIndex delayedIndex = + DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L) + .build(); listA2.add(delayedIndex); } - SnapshotSegment snapshotSegmentA2 = new SnapshotSegment(); - snapshotSegmentA2.addAllIndexes(listA2); + SnapshotSegment snapshotSegmentA2 = SnapshotSegment.newBuilder().addAllIndexes(listA2).build(); List segmentListA = new ArrayList<>(); segmentListA.add(snapshotSegmentA1); @@ -75,32 +85,36 @@ public void testCombinedSegmentDelayedIndexQueue() { List listB = new ArrayList<>(); for (int i = 0; i < 9; i++) { - DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L); + DelayedIndex delayedIndex = + DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) + .build(); - DelayedIndex delayedIndex2 = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L); + DelayedIndex delayedIndex2 = + DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) + .build(); listB.add(delayedIndex); listB.add(delayedIndex2); } - SnapshotSegment snapshotSegmentB = new SnapshotSegment(); - snapshotSegmentB.addAllIndexes(listB); + SnapshotSegment snapshotSegmentB = SnapshotSegment.newBuilder().addAllIndexes(listB).build(); List segmentListB = new ArrayList<>(); segmentListB.add(snapshotSegmentB); - segmentListB.add(new SnapshotSegment()); + segmentListB.add(SnapshotSegment.newBuilder().build()); List listC = new ArrayList<>(); for (int i = 10; i < 30; i+=2) { DelayedIndex delayedIndex = - new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L); + DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) + .build(); DelayedIndex delayedIndex2 = - new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L); + DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) + .build(); listC.add(delayedIndex); listC.add(delayedIndex2); } - SnapshotSegment snapshotSegmentC = new SnapshotSegment(); - snapshotSegmentC.addAllIndexes(listC); + SnapshotSegment snapshotSegmentC = SnapshotSegment.newBuilder().addAllIndexes(listC).build(); List segmentListC = new ArrayList<>(); segmentListC.add(snapshotSegmentC); @@ -109,14 +123,11 @@ public void testCombinedSegmentDelayedIndexQueue() { int count = 0; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex pop = new DelayedIndex(); - delayedIndexQueue.popToObject(pop); + DelayedIndex pop = delayedIndexQueue.pop(); log.info("{} , {}, {}", pop.getTimestamp(), pop.getLedgerId(), pop.getEntryId()); count++; if (!delayedIndexQueue.isEmpty()) { DelayedIndex peek = delayedIndexQueue.peek(); - long timestamp = delayedIndexQueue.peekTimestamp(); - Assert.assertEquals(timestamp, peek.getTimestamp()); Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } @@ -136,13 +147,10 @@ public void TripleLongPriorityDelayedIndexQueueTest() { int count = 0; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex pop = new DelayedIndex(); - delayedIndexQueue.popToObject(pop); + DelayedIndex pop = delayedIndexQueue.pop(); count++; if (!delayedIndexQueue.isEmpty()) { DelayedIndex peek = delayedIndexQueue.peek(); - long timestamp = delayedIndexQueue.peekTimestamp(); - Assert.assertEquals(timestamp, peek.getTimestamp()); Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } From 2fa28f427b8e5e05b4d0a233cc68f62d5bbe34d7 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sat, 22 Apr 2023 13:28:53 -0700 Subject: [PATCH 022/494] [improve] Allow to build and push multi-arch Docker images (#19432) Co-authored-by: Lari Hotari Co-authored-by: Yong Zhang Co-authored-by: Zixuan Liu Co-authored-by: tison (cherry picked from commit 4190e40ab4ac448a2f94edd1c621dc1c67d9ee4b) --- .github/workflows/pulsar-ci.yaml | 1 + build/build_java_test_image.sh | 1 + build/pulsar_ci_tool.sh | 3 +- docker/pulsar-all/pom.xml | 42 ++++++++++++--- docker/pulsar/pom.xml | 51 +++++++++++++++---- pom.xml | 8 ++- .../latest-version-image/pom.xml | 6 +++ 7 files changed, 94 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index f7dbc755264d6..4d4f2902d8f27 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -798,6 +798,7 @@ jobs: # build docker image # include building of Pulsar SQL, Connectors, Offloaders and server distros mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ + -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true diff --git a/build/build_java_test_image.sh b/build/build_java_test_image.sh index 0747e6dacb82a..459bf26f98eff 100755 --- a/build/build_java_test_image.sh +++ b/build/build_java_test_image.sh @@ -27,5 +27,6 @@ if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]] SQUASH_PARAM="-Ddocker.squash=true" fi mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \ + -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \ "$@" install \ No newline at end of file diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh index 61199eda2c5d8..d946edd395789 100755 --- a/build/pulsar_ci_tool.sh +++ b/build/pulsar_ci_tool.sh @@ -46,7 +46,8 @@ function ci_print_thread_dumps() { # runs maven function _ci_mvn() { - mvn -B -ntp "$@" + mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ + "$@" } # runs OWASP Dependency Check for all projects diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index b4186c8c1a9a7..e78eebecc7abf 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -68,10 +68,6 @@ docker - - target/apache-pulsar-io-connectors-${project.version}-bin - target/pulsar-offloader-distribution-${project.version}-bin.tar.gz - @@ -143,17 +139,25 @@ - pulsar-all + ${docker.organization}/pulsar-all ${project.basedir} latest + ${project.version} + + target/apache-pulsar-io-connectors-${project.version}-bin + target/pulsar-offloader-distribution-${project.version}-bin.tar.gz + + + + ${docker.platforms} + + - latest - ${docker.organization} @@ -161,5 +165,29 @@ + + + docker-push + + + + io.fabric8 + docker-maven-plugin + + + default + package + + build + tag + push + + + + + + + + diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index d83755846b460..5e849f2887e53 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -47,15 +47,14 @@ + + mirror://mirrors.ubuntu.com/mirrors.txt + http://security.ubuntu.com/ubuntu/ + + docker - - target/pulsar-server-distribution-${project.version}-bin.tar.gz - ${pulsar.client.python.version} - ${env.UBUNTU_MIRROR} - ${env.UBUNTU_SECURITY_MIRROR} - @@ -72,17 +71,27 @@ - pulsar + ${docker.organization}/pulsar + + target/pulsar-server-distribution-${project.version}-bin.tar.gz + ${pulsar.client.python.version} + ${UBUNTU_MIRROR} + ${UBUNTU_SECURITY_MIRROR} + ${project.basedir} latest + ${project.version} + + + ${docker.platforms} + + - latest - ${docker.organization} @@ -108,5 +117,29 @@ + + + docker-push + + + + io.fabric8 + docker-maven-plugin + + + default + package + + build + tag + push + + + + + + + + diff --git a/pom.xml b/pom.xml index f960c694c1905..ab87391c68808 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,12 @@ flexible messaging model and an intuitive client API. UTF-8 2023-04-12T05:18:48Z true + + + + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED @@ -152,7 +158,7 @@ flexible messaging model and an intuitive client API. 0.10.2 1.6.2 8.37 - 0.40.2 + 0.42.1 true 0.5.0 3.19.6 diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 82e966230241c..1b37bc2190c9b 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -147,6 +147,7 @@ package build + tag @@ -159,6 +160,11 @@ ${project.version} true + + + ${docker.platforms} + + From 89e4d46f4a5585ac126a03aa298020203100ad71 Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Sun, 23 Apr 2023 11:27:48 +0800 Subject: [PATCH 023/494] [fix] [broker] Make `LeastResourceUsageWithWeight` thread safe (#20159) Fix #20160 ### Motivation LeastResourceUsageWithWeight is an stateful object and current code will be accessed by multithread. thread 1: is execute https://github.com/apache/pulsar/blob/2b41e4eafb1cba0e548dd90df60e8cdbb24cd490/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java#L89-L91 and thread 2: is execute https://github.com/apache/pulsar/blob/2b41e4eafb1cba0e548dd90df60e8cdbb24cd490/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java#L147-L150 so an IndexOutOfBound occurs. ### Modifications change the state to `ThreadLocal` (cherry picked from commit 963260abfa142b8cf9ffe372d85d470a78c17235) --- .../strategy/LeastResourceUsageWithWeight.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java index 902cfdaf73f5f..98986d84b9858 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/strategy/LeastResourceUsageWithWeight.java @@ -23,6 +23,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; +import javax.annotation.concurrent.ThreadSafe; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; @@ -35,14 +36,15 @@ * cause cluster fluctuations due to short-term load jitter. */ @Slf4j +@ThreadSafe public class LeastResourceUsageWithWeight implements BrokerSelectionStrategy { // Maintain this list to reduce object creation. - private final ArrayList bestBrokers; - private final Set noLoadDataBrokers; + private final ThreadLocal> bestBrokers; + private final ThreadLocal> noLoadDataBrokers; public LeastResourceUsageWithWeight() { - this.bestBrokers = new ArrayList<>(); - this.noLoadDataBrokers = new HashSet<>(); + this.bestBrokers = ThreadLocal.withInitial(ArrayList::new); + this.noLoadDataBrokers = ThreadLocal.withInitial(HashSet::new); } // A broker's max resource usage with weight using its historical load and short-term load data with weight. @@ -70,7 +72,6 @@ private double getMaxResourceUsageWithWeight(final String broker, final BrokerLo /** * Find a suitable broker to assign the given bundle to. - * This method is not thread safety. * * @param candidates The candidates for which the bundle may be assigned. * @param bundleToAssign The data for the bundle to assign. @@ -86,6 +87,9 @@ public Optional select( return Optional.empty(); } + ArrayList bestBrokers = this.bestBrokers.get(); + HashSet noLoadDataBrokers = this.noLoadDataBrokers.get(); + bestBrokers.clear(); noLoadDataBrokers.clear(); // Maintain of list of all the best scoring brokers and then randomly @@ -135,9 +139,7 @@ public Optional select( log.info("Assign randomly as none of the brokers are underloaded. candidatesSize:{}, " + "noLoadDataBrokersSize:{}", candidates.size(), noLoadDataBrokers.size()); } - for (String broker : candidates) { - bestBrokers.add(broker); - } + bestBrokers.addAll(candidates); } if (debugMode) { From 49539c20bea1d8b10b8465be5ee632c1c4bdc7ce Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 24 Apr 2023 11:46:27 +0200 Subject: [PATCH 024/494] Revert "[improve][broker] Cache LedgerHandle in BookkeeperBucketSnapshotStorage (#20117)" This reverts commit e1d63990644700bf61b3d7af1ef6d4d62145c2bb. --- .../BookkeeperBucketSnapshotStorage.java | 52 +++++++++---------- .../BookkeeperBucketSnapshotStorageTest.java | 43 --------------- 2 files changed, 26 insertions(+), 69 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 18a4c322f7b27..9c30ccf1c0b7e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import javax.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; @@ -49,8 +48,6 @@ public class BookkeeperBucketSnapshotStorage implements BucketSnapshotStorage { private final ServiceConfiguration config; private BookKeeper bookKeeper; - private final Map> ledgerHandleFutureCache = new ConcurrentHashMap<>(); - public BookkeeperBucketSnapshotStorage(PulsarService pulsar) { this.pulsar = pulsar; this.config = pulsar.getConfig(); @@ -69,30 +66,45 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - return getLedgerHandle(bucketId).thenCompose(ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0) - .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture snapshotFuture = + getLedgerEntry(ledgerHandle, 0, 0) + .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())); + + snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return snapshotFuture; + }); } @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { - return getLedgerHandle(bucketId).thenCompose( - ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) - .thenApply(this::parseSnapshotSegmentEntries)); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture> parseFuture = + getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) + .thenApply(this::parseSnapshotSegmentEntries); + + parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return parseFuture; + }); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return getLedgerHandle(bucketId).thenCompose( - ledgerHandle -> CompletableFuture.completedFuture(ledgerHandle.getLength())); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture lengthFuture = + CompletableFuture.completedFuture(ledgerHandle.getLength()); + + lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return lengthFuture; + }); } @Override public CompletableFuture deleteBucketSnapshot(long bucketId) { - CompletableFuture ledgerHandleFuture = ledgerHandleFutureCache.remove(bucketId); - if (ledgerHandleFuture != null) { - ledgerHandleFuture.whenComplete((lh, ex) -> closeLedger(lh)); - } return deleteLedger(bucketId); } @@ -166,18 +178,6 @@ private CompletableFuture createLedger(String bucketKey, String to return future; } - private CompletableFuture getLedgerHandle(Long ledgerId) { - CompletableFuture ledgerHandleCompletableFuture = - ledgerHandleFutureCache.computeIfAbsent(ledgerId, k -> openLedger(ledgerId)); - // remove future of completed exceptionally - ledgerHandleCompletableFuture.whenComplete((__, ex) -> { - if (ex != null) { - ledgerHandleFutureCache.remove(ledgerId, ledgerHandleCompletableFuture); - } - }); - return ledgerHandleCompletableFuture; - } - private CompletableFuture openLedger(Long ledgerId) { final CompletableFuture future = new CompletableFuture<>(); bookKeeper.asyncOpenLedger( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index 7cb6b8d5865bb..a628b58e10d32 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -30,7 +30,6 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; -import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -205,46 +204,4 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted Assert.assertTrue(bucketSnapshotLength > 0L); } - @Test - public void testConcurrencyGet() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() - .setMinScheduleTimestamp(System.currentTimeMillis()) - .setMaxScheduleTimestamp(System.currentTimeMillis()) - .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() - .addMetadataList(segmentMetadata) - .build(); - List bucketSnapshotSegments = new ArrayList<>(); - - long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); - bucketSnapshotSegments.add(snapshotSegment); - bucketSnapshotSegments.add(snapshotSegment); - - CompletableFuture future = - bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, - bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); - Long bucketId = future.get(); - Assert.assertNotNull(bucketId); - - List> futures = new ArrayList<>(); - for (int i = 0; i < 100; i++) { - CompletableFuture future0 = CompletableFuture.runAsync(() -> { - List list = - bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join(); - Assert.assertTrue(list.size() > 0); - }); - futures.add(future0); - } - - FutureUtil.waitForAll(futures).join(); - } - } From f3572574b132dfb6d5a67011f7fb68096e69e1c7 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 24 Apr 2023 11:48:03 +0200 Subject: [PATCH 025/494] Revert "[improve][broker] Optimize delayed metadata index bitmap (#20136)" This reverts commit ff59240165c73a9c3a3dcca20702ab44b0b18d33. --- .../bucket/BucketDelayedDeliveryTracker.java | 3 -- .../delayed/bucket/ImmutableBucket.java | 53 ++++++++----------- .../broker/delayed/bucket/MutableBucket.java | 11 ++-- 3 files changed, 25 insertions(+), 42 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index b17387e276e2b..b4d1745e22f3a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -485,9 +485,6 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List> asyncLoadNextBucketSnapshotEntry(b this.setLastSegmentEntryId(metadataList.size()); this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); List firstScheduleTimestamps = metadataList.stream().map( - SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); + SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); this.setFirstScheduleTimestamps(firstScheduleTimestamps); return nextSnapshotEntryIndex + 1; @@ -138,37 +139,25 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b }); } - /** - * Recover delayed index bit map and message numbers. - * @throws InvalidRoaringFormat invalid bitmap serialization format - */ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, - List segmentMetaList) { - delayedIndexBitMap.clear(); // cleanup dirty bm - final var numberMessages = new MutableLong(0); - for (int i = startSnapshotIndex; i < segmentMetaList.size(); i++) { - for (final var entry : segmentMetaList.get(i).getDelayedIndexBitMapMap().entrySet()) { - final var ledgerId = entry.getKey(); - final var bs = entry.getValue(); - final var sbm = new RoaringBitmap(); - try { - sbm.deserialize(bs.asReadOnlyByteBuffer()); - } catch (IOException e) { - throw new InvalidRoaringFormat(e.getMessage()); + List segmentMetadata) { + this.delayedIndexBitMap.clear(); + MutableLong numberMessages = new MutableLong(0); + for (int i = startSnapshotIndex; i < segmentMetadata.size(); i++) { + Map bitByteStringMap = segmentMetadata.get(i).getDelayedIndexBitMapMap(); + bitByteStringMap.forEach((leaderId, bitSetString) -> { + boolean exist = this.delayedIndexBitMap.containsKey(leaderId); + RoaringBitmap bitSet = + new ImmutableRoaringBitmap(bitSetString.asReadOnlyByteBuffer()).toRoaringBitmap(); + numberMessages.add(bitSet.getCardinality()); + if (!exist) { + this.delayedIndexBitMap.put(leaderId, bitSet); + } else { + this.delayedIndexBitMap.get(leaderId).or(bitSet); } - numberMessages.add(sbm.getCardinality()); - delayedIndexBitMap.compute(ledgerId, (lId, bm) -> { - if (bm == null) { - return sbm; - } - bm.or(sbm); - return bm; - }); - } + }); } - // optimize bm - delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize); - setNumberBucketDelayedMessages(numberMessages.getValue()); + this.setNumberBucketDelayedMessages(numberMessages.getValue()); } CompletableFuture> getRemainSnapshotSegment() { @@ -204,7 +193,7 @@ CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete); } else { log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey); + dispatcherName, bucketId, bucketKey); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete, System.currentTimeMillis() - deleteStartTime); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index f404d5d02c15a..e49ebe9606e01 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -116,13 +116,10 @@ Pair createImmutableBucketAndAsyncPersistent( Iterator> iterator = bitMap.entrySet().iterator(); while (iterator.hasNext()) { - final var entry = iterator.next(); - final var lId = entry.getKey(); - final var bm = entry.getValue(); - bm.runOptimize(); - final var array = new byte[bm.serializedSizeInBytes()]; - bm.serialize(ByteBuffer.wrap(array)); - segmentMetadataBuilder.putDelayedIndexBitMap(lId, ByteString.copyFrom(array)); + Map.Entry entry = iterator.next(); + byte[] array = new byte[entry.getValue().serializedSizeInBytes()]; + entry.getValue().serialize(ByteBuffer.wrap(array)); + segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), ByteString.copyFrom(array)); iterator.remove(); } From 610c5f151436557d51b56e06a97ba422553592d7 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 24 Apr 2023 11:49:13 +0200 Subject: [PATCH 026/494] Revert "[improve][broker] Make timer execute immediately after load index (#20126)" This reverts commit 49480ea558e647169e8df01bfd2e871a5386e19e. --- .../delayed/bucket/BucketDelayedDeliveryTracker.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index b4d1745e22f3a..f57248acbb7bd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -541,7 +541,7 @@ public long getBufferMemoryUsage() { @Override public synchronized NavigableSet getScheduledMessages(int maxMessages) { - if (!checkPendingLoadDone()) { + if (!checkPendingOpDone()) { if (log.isDebugEnabled()) { log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.", dispatcher.getName()); @@ -628,11 +628,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa if (timeout != null) { timeout.cancel(); } - timeout = timer.newTimeout(this, 0, TimeUnit.MILLISECONDS); + timeout = timer.newTimeout(this, tickTimeMillis, TimeUnit.MILLISECONDS); } }); - if (!checkPendingLoadDone() || loadFuture.isCompletedExceptionally()) { + if (!checkPendingOpDone() || loadFuture.isCompletedExceptionally()) { break; } } @@ -651,7 +651,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa return positions; } - private synchronized boolean checkPendingLoadDone() { + private synchronized boolean checkPendingOpDone() { if (pendingLoad == null || pendingLoad.isDone()) { pendingLoad = null; return true; From 254f09b70d80640095f2fe6d12ed79f046fb3241 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 24 Apr 2023 11:50:50 +0200 Subject: [PATCH 027/494] Revert "[improve] [broker] Close temporary open ledger in BookkeeperBucketSnapshotStorage (#20111)" This reverts commit 1d1a3ef864a65c995ceda4b7875ed934c2574298. --- .../BookkeeperBucketSnapshotStorage.java | 33 ++++--------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 9c30ccf1c0b7e..e7d4f9301dd36 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -66,41 +66,22 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture snapshotFuture = - getLedgerEntry(ledgerHandle, 0, 0) - .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())); - - snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return snapshotFuture; - }); + return openLedger(bucketId).thenCompose( + ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0). + thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); } @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture> parseFuture = - getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) - .thenApply(this::parseSnapshotSegmentEntries); - - parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return parseFuture; - }); + return openLedger(bucketId).thenCompose( + ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, + lastSegmentEntryId).thenApply(this::parseSnapshotSegmentEntries)); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture lengthFuture = - CompletableFuture.completedFuture(ledgerHandle.getLength()); - - lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return lengthFuture; - }); + return openLedger(bucketId).thenApply(LedgerHandle::getLength); } @Override From e8261fcef8d76061d378d8a9f3b05a5d115013fe Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Mon, 24 Apr 2023 12:10:50 +0200 Subject: [PATCH 028/494] Revert "[fix][broker] Fix entry filter feature for the non-persistent topic (#20141)" This reverts commit e27abe9e128fb71b65ffe06417574c9a7f3facbd. --- .../nonpersistent/NonPersistentTopic.java | 5 +-- .../service/plugin/FilterEntryTest.java | 15 ++----- .../broker/stats/SubscriptionStatsTest.java | 42 +++++++++---------- 3 files changed, 26 insertions(+), 36 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 33258b06726b5..317b8df6b9a82 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -26,7 +26,7 @@ import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -199,8 +199,7 @@ public void publishMessage(ByteBuf data, PublishContext callback) { // entry internally retains data so, duplicateBuffer should be release here duplicateBuffer.release(); if (subscription.getDispatcher() != null) { - // Dispatcher needs to call the set method to support entry filter feature. - subscription.getDispatcher().sendMessages(Arrays.asList(entry)); + subscription.getDispatcher().sendMessages(Collections.singletonList(entry)); } else { // it happens when subscription is created but dispatcher is not created as consumer is not added // yet diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java index b868858646c50..4b9d91fbde219 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java @@ -51,7 +51,6 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.EntryFilterSupport; -import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -287,16 +286,10 @@ public void testFilter() throws Exception { } - @DataProvider(name = "topicProvider") - public Object[][] topicProvider() { - return new Object[][]{ - {"persistent://prop/ns-abc/topic" + UUID.randomUUID()}, - {"non-persistent://prop/ns-abc/topic" + UUID.randomUUID()}, - }; - } - @Test(dataProvider = "topicProvider") - public void testFilteredMsgCount(String topic) throws Throwable { + @Test + public void testFilteredMsgCount() throws Throwable { + String topic = "persistent://prop/ns-abc/topic" + UUID.randomUUID(); String subName = "sub"; try (Producer producer = pulsarClient.newProducer(Schema.STRING) @@ -305,7 +298,7 @@ public void testFilteredMsgCount(String topic) throws Throwable { .subscriptionName(subName).subscribe()) { // mock entry filters - Subscription subscription = pulsar.getBrokerService() + PersistentSubscription subscription = (PersistentSubscription) pulsar.getBrokerService() .getTopicReference(topic).get().getSubscription(subName); Dispatcher dispatcher = subscription.getDispatcher(); Field field = EntryFilterSupport.class.getDeclaredField("entryFilters"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java index d5e0066a86f15..bf9c1d540bf87 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java @@ -211,22 +211,21 @@ public void testSubscriptionStats(final String topic, final String subName, bool hasFilterField.set(dispatcher, true); } - int rejectedCount = 100; - int acceptCount = 100; - int scheduleCount = 100; - for (int i = 0; i < rejectedCount; i++) { - producer.newMessage().property("REJECT", " ").value(UUID.randomUUID().toString()).send(); - } - for (int i = 0; i < acceptCount; i++) { + for (int i = 0; i < 100; i++) { producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send(); } - for (int i = 0; i < scheduleCount; i++) { + for (int i = 0; i < 100; i++) { + producer.newMessage().property("REJECT", " ").value(UUID.randomUUID().toString()).send(); + } + for (int i = 0; i < 100; i++) { producer.newMessage().property("RESCHEDULE", " ").value(UUID.randomUUID().toString()).send(); } - for (int i = 0; i < acceptCount; i++) { - Message message = consumer.receive(1, TimeUnit.SECONDS); - Assert.assertNotNull(message); + for (;;) { + Message message = consumer.receive(10, TimeUnit.SECONDS); + if (message == null) { + break; + } consumer.acknowledge(message); } @@ -264,12 +263,12 @@ public void testSubscriptionStats(final String topic, final String subName, bool .mapToDouble(m-> m.value).sum(); if (setFilter) { - Assert.assertEquals(filterAccepted, acceptCount); - Assert.assertEquals(filterRejected, rejectedCount); - // Only works on the test, if there are some markers, - // the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount - Assert.assertEquals(throughFilter, - filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter); + Assert.assertEquals(filterAccepted, 100); + if (isPersistent) { + Assert.assertEquals(filterRejected, 100); + // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount + Assert.assertEquals(throughFilter, filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter); + } } else { Assert.assertEquals(throughFilter, 0D); Assert.assertEquals(filterAccepted, 0D); @@ -283,20 +282,19 @@ public void testSubscriptionStats(final String topic, final String subName, bool Assert.assertEquals(rescheduledMetrics.size(), 0); } - testSubscriptionStatsAdminApi(topic, subName, setFilter, acceptCount, rejectedCount); + testSubscriptionStatsAdminApi(topic, subName, setFilter); } - private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter, - int acceptCount, int rejectedCount) throws Exception { + private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter) throws Exception { boolean persistent = TopicName.get(topic).isPersistent(); TopicStats topicStats = admin.topics().getStats(topic); SubscriptionStats stats = topicStats.getSubscriptions().get(subName); Assert.assertNotNull(stats); if (setFilter) { - Assert.assertEquals(stats.getFilterAcceptedMsgCount(), acceptCount); + Assert.assertEquals(stats.getFilterAcceptedMsgCount(), 100); if (persistent) { - Assert.assertEquals(stats.getFilterRejectedMsgCount(), rejectedCount); + Assert.assertEquals(stats.getFilterRejectedMsgCount(), 100); // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount Assert.assertEquals(stats.getFilterProcessedMsgCount(), stats.getFilterAcceptedMsgCount() + stats.getFilterRejectedMsgCount() From 7636e8989f4d3fc24fce69a356d54e1c550945ed Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Sat, 22 Apr 2023 10:58:05 +0800 Subject: [PATCH 029/494] [fix][client] Fix breaking changes for the deprecated methods of TopicMessageIdImpl (#20163) (cherry picked from commit 6036dcce8d48f8e1ae311153eeb625f51c5aa827) --- .../apache/pulsar/client/impl/TopicMessageIdImpl.java | 2 +- .../pulsar/client/impl/TopicMessageIdImplTest.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java index 00fe12b62b194..3dc9b23e93e86 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicMessageIdImpl.java @@ -62,7 +62,7 @@ public String getTopicPartitionName() { @Deprecated public MessageId getInnerMessageId() { - return new MessageIdImpl(getLedgerId(), getEntryId(), getPartitionIndex()); + return msgId; } @Override diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java index daf49f0e7752e..1ddd47af91dff 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicMessageIdImplTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertSame; import org.testng.annotations.Test; @@ -54,4 +55,12 @@ public void equalsTest() { assertNotEquals(topicMsgId1, topicMsgId2); } + @Test + public void testDeprecatedMethods() { + BatchMessageIdImpl msgId = new BatchMessageIdImpl(1, 2, 3, 4); + TopicMessageIdImpl topicMsgId = new TopicMessageIdImpl("topic-partition-0", "topic", msgId); + assertSame(topicMsgId.getInnerMessageId(), msgId); + assertEquals(topicMsgId.getTopicPartitionName(), topicMsgId.getOwnerTopic()); + assertEquals(topicMsgId.getTopicName(), "topic"); + } } From 18e48bfedf3a1bee9bc36da15a1b9378c585d06e Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 5 May 2023 15:42:03 +0800 Subject: [PATCH 030/494] [improve][build] Upgrade dependencies to reduce CVE (#20227) --- .../server/src/assemble/LICENSE.bin.txt | 52 +++++++++---------- pom.xml | 6 +-- pulsar-sql/presto-distribution/LICENSE | 34 ++++++------ src/owasp-dependency-check-suppressions.xml | 30 ----------- 4 files changed, 46 insertions(+), 76 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index c5e69f5cde97a..3a30e86904dce 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -271,9 +271,9 @@ The Apache Software License, Version 2.0 * J2ObjC Annotations -- com.google.j2objc-j2objc-annotations-1.3.jar * Netty Reactive Streams -- com.typesafe.netty-netty-reactive-streams-2.0.6.jar * Swagger - - io.swagger-swagger-annotations-1.6.2.jar - - io.swagger-swagger-core-1.6.2.jar - - io.swagger-swagger-models-1.6.2.jar + - io.swagger-swagger-annotations-1.6.10.jar + - io.swagger-swagger-core-1.6.10.jar + - io.swagger-swagger-models-1.6.10.jar * DataSketches - com.yahoo.datasketches-memory-0.8.3.jar - com.yahoo.datasketches-sketches-core-0.8.3.jar @@ -383,25 +383,25 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.12.1.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar * Jetty - - org.eclipse.jetty-jetty-client-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-continuation-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-http-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-io-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-proxy-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-security-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-server-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-servlet-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-servlets-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-util-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-util-ajax-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-api-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-client-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-common-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-server-9.4.48.v20220622.jar - - org.eclipse.jetty.websocket-websocket-servlet-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar - - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar + - org.eclipse.jetty-jetty-client-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-continuation-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-http-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-io-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-proxy-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-security-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-server-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-servlet-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-servlets-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-util-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-util-ajax-9.4.51.v20230217.jar + - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.51.v20230217.jar + - org.eclipse.jetty.websocket-websocket-api-9.4.51.v20230217.jar + - org.eclipse.jetty.websocket-websocket-client-9.4.51.v20230217.jar + - org.eclipse.jetty.websocket-websocket-common-9.4.51.v20230217.jar + - org.eclipse.jetty.websocket-websocket-server-9.4.51.v20230217.jar + - org.eclipse.jetty.websocket-websocket-servlet-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-alpn-server-9.4.51.v20230217.jar * SnakeYaml -- org.yaml-snakeyaml-1.32.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar @@ -412,10 +412,10 @@ The Apache Software License, Version 2.0 * Okio - com.squareup.okio-okio-2.8.0.jar * Javassist -- org.javassist-javassist-3.25.0-GA.jar * Kotlin Standard Lib - - org.jetbrains.kotlin-kotlin-stdlib-1.4.32.jar - - org.jetbrains.kotlin-kotlin-stdlib-common-1.4.32.jar - - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.4.32.jar - - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.4.32.jar + - org.jetbrains.kotlin-kotlin-stdlib-1.6.0.jar + - org.jetbrains.kotlin-kotlin-stdlib-common-1.6.0.jar + - org.jetbrains.kotlin-kotlin-stdlib-jdk7-1.6.0.jar + - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.6.0.jar - org.jetbrains-annotations-13.0.jar * gRPC - io.grpc-grpc-all-1.45.1.jar diff --git a/pom.xml b/pom.xml index ab87391c68808..e7e00db74e4db 100644 --- a/pom.xml +++ b/pom.xml @@ -141,7 +141,7 @@ flexible messaging model and an intuitive client API. 5.1.0 4.1.89.Final 0.0.18.Final - 9.4.48.v20220622 + 9.4.51.v20230217 2.5.2 2.34 1.10.50 @@ -156,7 +156,7 @@ flexible messaging model and an intuitive client API. 1.0.2.3 2.13.4.20221013 0.10.2 - 1.6.2 + 1.6.10 8.37 0.42.1 true @@ -235,7 +235,7 @@ flexible messaging model and an intuitive client API. 2.8.0 - 1.4.32 + 1.6.0 1.0 9.1.6 5.3.26 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index ede2ea3999957..9a5e4678baadc 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -279,22 +279,22 @@ The Apache Software License, Version 2.0 - joda-time-2.10.10.jar - failsafe-2.4.4.jar * Jetty - - http2-client-9.4.48.v20220622.jar - - http2-common-9.4.48.v20220622.jar - - http2-hpack-9.4.48.v20220622.jar - - http2-http-client-transport-9.4.48.v20220622.jar - - jetty-alpn-client-9.4.48.v20220622.jar - - http2-server-9.4.48.v20220622.jar - - jetty-alpn-java-client-9.4.48.v20220622.jar - - jetty-client-9.4.48.v20220622.jar - - jetty-http-9.4.48.v20220622.jar - - jetty-io-9.4.48.v20220622.jar - - jetty-jmx-9.4.48.v20220622.jar - - jetty-security-9.4.48.v20220622.jar - - jetty-server-9.4.48.v20220622.jar - - jetty-servlet-9.4.48.v20220622.jar - - jetty-util-9.4.48.v20220622.jar - - jetty-util-ajax-9.4.48.v20220622.jar + - http2-client-9.4.51.v20230217.jar + - http2-common-9.4.51.v20230217.jar + - http2-hpack-9.4.51.v20230217.jar + - http2-http-client-transport-9.4.51.v20230217.jar + - jetty-alpn-client-9.4.51.v20230217.jar + - http2-server-9.4.51.v20230217.jar + - jetty-alpn-java-client-9.4.51.v20230217.jar + - jetty-client-9.4.51.v20230217.jar + - jetty-http-9.4.51.v20230217.jar + - jetty-io-9.4.51.v20230217.jar + - jetty-jmx-9.4.51.v20230217.jar + - jetty-security-9.4.51.v20230217.jar + - jetty-server-9.4.51.v20230217.jar + - jetty-servlet-9.4.51.v20230217.jar + - jetty-util-9.4.51.v20230217.jar + - jetty-util-ajax-9.4.51.v20230217.jar * Byte Buddy - byte-buddy-1.11.13.jar * Apache BVal @@ -473,7 +473,7 @@ The Apache Software License, Version 2.0 * Apache Yetus Audience Annotations - audience-annotations-0.12.0.jar * Swagger - - swagger-annotations-1.6.2.jar + - swagger-annotations-1.6.10.jar * Perfmark - perfmark-api-0.19.0.jar * Annotations diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 84ed2a3332cd4..4bca86ab12620 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -72,36 +72,6 @@ CVE-2022-23712 - - - - ef50bfa2c0491a11dcc35d9822edbfd6170e1ea2 - cpe:/a:jetbrains:kotlin - - - - 3546900a3ebff0c43f31190baf87a9220e37b7ea - CVE-2022-24329 - - - - 3302f9ec8a5c1ed220781dbd37770072549bd333 - CVE-2022-24329 - - - - 461367948840adbb0839c51d91ed74ef4a9ccb52 - CVE-2022-24329 - - Date: Sun, 7 May 2023 17:20:46 +0800 Subject: [PATCH 031/494] upgrade spring to 5.3.27 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e7e00db74e4db..252cedbf65927 100644 --- a/pom.xml +++ b/pom.xml @@ -238,7 +238,7 @@ flexible messaging model and an intuitive client API. 1.6.0 1.0 9.1.6 - 5.3.26 + 5.3.27 4.5.13 4.4.15 0.5.11 From 7cb5ab701825a76fa868da6ca37c6a825692d2dc Mon Sep 17 00:00:00 2001 From: Jiwe Guo Date: Sun, 7 May 2023 19:44:28 +0800 Subject: [PATCH 032/494] upgrade hadoop2 version --- pom.xml | 2 +- pulsar-io/flume/pom.xml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 252cedbf65927..8f2d78af042ae 100644 --- a/pom.xml +++ b/pom.xml @@ -198,7 +198,7 @@ flexible messaging model and an intuitive client API. 1.15.16.Final 0.11.1 0.28.0 - 2.10.2 + 3.3.4 3.3.4 2.4.15 31.0.1-jre diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 5df8971e11790..358c25d6e88f8 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -65,6 +65,10 @@ avro org.apache.avro + + commons-collections + commons-collections + From aa3c319d45bd3a9e08dd244eefd328f08a0e3479 Mon Sep 17 00:00:00 2001 From: Jiwe Guo Date: Sun, 7 May 2023 19:47:40 +0800 Subject: [PATCH 033/494] Revert "upgrade hadoop2 version" This reverts commit 7cb5ab701825a76fa868da6ca37c6a825692d2dc. --- pom.xml | 2 +- pulsar-io/flume/pom.xml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 8f2d78af042ae..252cedbf65927 100644 --- a/pom.xml +++ b/pom.xml @@ -198,7 +198,7 @@ flexible messaging model and an intuitive client API. 1.15.16.Final 0.11.1 0.28.0 - 3.3.4 + 2.10.2 3.3.4 2.4.15 31.0.1-jre diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 358c25d6e88f8..5df8971e11790 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -65,10 +65,6 @@ avro org.apache.avro - - commons-collections - commons-collections - From b77cb5e748307fd8e9a13b54e37fd201c3e2592a Mon Sep 17 00:00:00 2001 From: Jiwe Guo Date: Sun, 7 May 2023 19:47:55 +0800 Subject: [PATCH 034/494] Revert "upgrade spring to 5.3.27" This reverts commit 6c926f04daafad6c0ae3832e49aca1a3b13b64f4. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 252cedbf65927..e7e00db74e4db 100644 --- a/pom.xml +++ b/pom.xml @@ -238,7 +238,7 @@ flexible messaging model and an intuitive client API. 1.6.0 1.0 9.1.6 - 5.3.27 + 5.3.26 4.5.13 4.4.15 0.5.11 From c7cffba6b623183f82ec48c6c83a098dcd1a1fa8 Mon Sep 17 00:00:00 2001 From: ken <1647023764@qq.com> Date: Thu, 13 Apr 2023 01:22:02 +0800 Subject: [PATCH 035/494] [fix] [broker] error TimeUnit to record publish latency (#20074) Co-authored-by: fanjianye Co-authored-by: tison (cherry picked from commit 547d792439e7b8bfefb0236b078ed145731da658) --- .../apache/pulsar/broker/rest/RestMessagePublishContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java index d7cee8b600b31..3c9adbd3e4fe4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/RestMessagePublishContext.java @@ -53,7 +53,7 @@ public void completed(Exception exception, long ledgerId, long entryId) { + "triggered send callback.", topic.getName(), ledgerId, entryId); } - topic.recordAddLatency(System.nanoTime() - startTimeNs, TimeUnit.MICROSECONDS); + topic.recordAddLatency(System.nanoTime() - startTimeNs, TimeUnit.NANOSECONDS); positionFuture.complete(PositionImpl.get(ledgerId, entryId)); } recycle(); From 9a271ae445471d8ba516abe53747eca0ae617b69 Mon Sep 17 00:00:00 2001 From: Andy Walker Date: Wed, 12 Apr 2023 20:28:42 -0400 Subject: [PATCH 036/494] [fix][fn] check user metric len before iterating (#20021) Co-authored-by: Andy Walker (cherry picked from commit 52e8144587548a692e550a8538f4d2667b5499d6) --- pulsar-function-go/pf/instance.go | 3 ++ .../pf/instanceControlServicer_test.go | 53 +++++++++++++++++++ pulsar-function-go/pf/stats_test.go | 27 ++++++++++ 3 files changed, 83 insertions(+) diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index 5d17cfe0c333a..a82273031ecfb 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -669,6 +669,9 @@ func (gi *goInstance) getTotalReceived1min() float32 { func (gi *goInstance) getUserMetricsMap() map[string]float64 { userMetricMap := map[string]float64{} filteredMetricFamilies := gi.getFilteredMetricFamilies(PulsarFunctionMetricsPrefix + UserMetric) + if len(filteredMetricFamilies) == 0 { + return userMetricMap + } for _, m := range filteredMetricFamilies[0].GetMetric() { var isFuncMetric bool var userLabelName string diff --git a/pulsar-function-go/pf/instanceControlServicer_test.go b/pulsar-function-go/pf/instanceControlServicer_test.go index 836ec6e5c7940..9344d0a591547 100644 --- a/pulsar-function-go/pf/instanceControlServicer_test.go +++ b/pulsar-function-go/pf/instanceControlServicer_test.go @@ -21,6 +21,7 @@ package pf import ( "context" + "fmt" "log" "net" "testing" @@ -76,3 +77,55 @@ func TestInstanceControlServicer_serve_creates_valid_instance(t *testing.T) { log.Printf("Response: %+v", resp.Success) assert.Equal(t, resp.Success, true) } + +func instanceCommunicationClient(t *testing.T, instance *goInstance) pb.InstanceControlClient { + t.Helper() + + if instance == nil { + t.Fatalf("cannot create communication client for nil instance") + } + + var ( + ctx context.Context = context.Background() + cf context.CancelFunc + ) + + if testDeadline, ok := t.Deadline(); ok { + ctx, cf = context.WithDeadline(context.Background(), testDeadline) + t.Cleanup(cf) + } + + lis = bufconn.Listen(bufSize) + t.Cleanup(func() { + lis.Close() + }) + // create a gRPC server object + grpcServer := grpc.NewServer() + t.Cleanup(func() { + grpcServer.Stop() + }) + + servicer := InstanceControlServicer{instance} + // must register before we start the service. + pb.RegisterInstanceControlServer(grpcServer, &servicer) + + // start the server + t.Logf("Serving InstanceCommunication on port %d", instance.context.GetPort()) + + go func() { + if err := grpcServer.Serve(lis); err != nil { + panic(fmt.Sprintf("grpc server exited with error: %v", err)) + } + }() + + // Now we can setup the client: + conn, err := grpc.DialContext(ctx, "bufnet", grpc.WithContextDialer(getBufDialer(lis)), grpc.WithInsecure()) + if err != nil { + t.Fatalf("Failed to dial bufnet: %v", err) + } + t.Cleanup(func() { + conn.Close() + }) + client := pb.NewInstanceControlClient(conn) + return client +} diff --git a/pulsar-function-go/pf/stats_test.go b/pulsar-function-go/pf/stats_test.go index d52b08b717377..7b415ef5eff0b 100644 --- a/pulsar-function-go/pf/stats_test.go +++ b/pulsar-function-go/pf/stats_test.go @@ -20,6 +20,7 @@ package pf import ( + "context" "fmt" "io/ioutil" "math" @@ -28,6 +29,7 @@ import ( "time" "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes/empty" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" @@ -257,3 +259,28 @@ func TestUserMetrics(t *testing.T) { gi.close() metricsServicer.close() } + +func TestInstanceControlMetrics(t *testing.T) { + instance := newGoInstance() + t.Cleanup(instance.close) + instanceClient := instanceCommunicationClient(t, instance) + _, err := instanceClient.GetMetrics(context.Background(), &empty.Empty{}) + assert.NoError(t, err, "err communicating with instance control: %v", err) + + testLabels := []string{"userMetricControlTest1", "userMetricControlTest2"} + for _, label := range testLabels { + assert.NotContainsf(t, label, "user metrics should not yet contain %s", label) + } + + for value, label := range testLabels { + instance.context.RecordMetric(label, float64(value+1)) + } + time.Sleep(time.Second) + + metrics, err := instanceClient.GetMetrics(context.Background(), &empty.Empty{}) + assert.NoError(t, err, "err communicating with instance control: %v", err) + for value, label := range testLabels { + assert.Containsf(t, metrics.UserMetrics, label, "user metrics should contain metric %s", label) + assert.EqualValuesf(t, value+1, metrics.UserMetrics[label], "user metric %s != %d", label, value+1) + } +} From 636465679175b07e260f2c668c8d4e4f4a683487 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 13 Apr 2023 23:09:10 +0800 Subject: [PATCH 037/494] [fix][test]Fix flaky test produceCommitTest (#20006) Fixes https://github.com/apache/pulsar/issues/18466 ### Motivation There are two main goals in solving this issue: 1. Fix the unstable tests in `produceCommitTest`. 2. Prevent transaction timeouts created by other tests from affecting the `testTxnTimeoutAtTransactionMetadataStore` test during its execution. ### Modification 1. Change the message-sending method to synchronous. (fix `produceCommitTest`) 2. Increase the transaction timeout to 10 minutes (fix `testTxnTimeoutAtTransactionMetadataStore`). (cherry picked from commit 653271e8d8aeb361ec260780ba88ccf3a880f403) --- .../apache/pulsar/client/impl/TransactionEndToEndTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 83feaa3ac1158..34cc3bc1ca526 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -280,9 +280,9 @@ private void produceCommitTest(boolean enableBatch) throws Exception { int messageCnt = 1000; for (int i = 0; i < messageCnt; i++) { if (i % 5 == 0) { - producer.newMessage(txn1).value(("Hello Txn - " + i).getBytes(UTF_8)).send(); + producer.newMessage(txn1).value(("Hello Txn - " + i).getBytes(UTF_8)).sendAsync(); } else { - producer.newMessage(txn2).value(("Hello Txn - " + i).getBytes(UTF_8)).send(); + producer.newMessage(txn2).value(("Hello Txn - " + i).getBytes(UTF_8)).sendAsync(); } txnMessageCnt++; } @@ -811,7 +811,7 @@ private void txnCumulativeAckTest(boolean batchEnable, int maxBatchSize, Subscri public Transaction getTxn() throws Exception { return pulsarClient .newTransaction() - .withTransactionTimeout(10, TimeUnit.SECONDS) + .withTransactionTimeout(10, TimeUnit.MINUTES) .build() .get(); } From b47af87a4ff4438549b7e9f3242ef5e3f7e33e11 Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Sun, 16 Apr 2023 17:05:46 +0300 Subject: [PATCH 038/494] [fix][doc] Add link to PIP proposal (#19911) ### Motivation Add a link to see existing PIP issues so you can select a number (cherry picked from commit 47b24b071353f9285704e29e397b851e31df4037) --- wiki/proposals/PIP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md index e10452b107fd8..159c9199895d1 100644 --- a/wiki/proposals/PIP.md +++ b/wiki/proposals/PIP.md @@ -80,7 +80,7 @@ The process works in the following way: 1. The author(s) of the proposal will create a GitHub issue ticket choosing the template for PIP proposals. The issue title should be "PIP-xxx: title", where the "xxx" number should be chosen to be the next number from the existing PIP - issues, listed [here]([url](https://github.com/apache/pulsar/labels/PIP)). + issues, listed [here](https://github.com/apache/pulsar/issues?q=is%3Aissue+label%3APIP+) 2. The author(s) will send a note to the dev@pulsar.apache.org mailing list to start the discussion, using subject prefix `[DISCUSS] PIP-xxx: {PIP TITLE}`. The discussion need to happen in the mailing list. Please avoid discussing it using From 9f98caa6032f9e8ae0fbd5aecb8a5b07713853b8 Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 17 Apr 2023 10:59:32 +0800 Subject: [PATCH 039/494] [fix][doc] ConnectorDocGenerator support Java 9+ (#20100) Signed-off-by: tison (cherry picked from commit b7eab9469177eda2c56e36bb9871aab48a17d4ec) --- .../pulsar/io/docs/ConnectorDocGenerator.java | 69 +++++++------------ .../io/elasticsearch/ElasticSearchConfig.java | 5 ++ 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java b/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java index fe12b2b11ce84..fec7b12087977 100644 --- a/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java +++ b/pulsar-io/docs/src/main/java/org/apache/pulsar/io/docs/ConnectorDocGenerator.java @@ -21,6 +21,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.google.common.base.Strings; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; @@ -28,11 +29,9 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.net.URL; -import java.net.URLClassLoader; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Set; import lombok.extern.slf4j.Slf4j; @@ -47,21 +46,12 @@ public class ConnectorDocGenerator { private static final String INDENT = " "; private static Reflections newReflections() throws Exception { - List urls = new ArrayList<>(); - ClassLoader[] classLoaders = new ClassLoader[] { - ConnectorDocGenerator.class.getClassLoader(), - Thread.currentThread().getContextClassLoader() - }; - for (int i = 0; i < classLoaders.length; i++) { - if (classLoaders[i] instanceof URLClassLoader) { - urls.addAll(Arrays.asList(((URLClassLoader) classLoaders[i]).getURLs())); - } else { - throw new RuntimeException("ClassLoader '" + classLoaders[i] + " is not an instance of URLClassLoader"); - } + final String[] classpathList = System.getProperty("java.class.path").split(":"); + final List urlList = new ArrayList<>(); + for (String file : classpathList) { + urlList.add(new File(file).toURI().toURL()); } - ConfigurationBuilder confBuilder = new ConfigurationBuilder(); - confBuilder.setUrls(urls); - return new Reflections(confBuilder); + return new Reflections(new ConfigurationBuilder().setUrls(urlList)); } private final Reflections reflections; @@ -70,7 +60,7 @@ public ConnectorDocGenerator() throws Exception { this.reflections = newReflections(); } - private void generateConnectorYaml(Class configClass, PrintWriter writer) { + private void generateConnectorYamlFile(Class configClass, PrintWriter writer) { log.info("Processing connector config class : {}", configClass); writer.println("configs:"); @@ -82,7 +72,9 @@ private void generateConnectorYaml(Class configClass, PrintWriter writer) { } FieldDoc fieldDoc = field.getDeclaredAnnotation(FieldDoc.class); if (null == fieldDoc) { - throw new RuntimeException("Missing `FieldDoc` for field '" + field.getName() + "'"); + final String message = "Missing FieldDoc for field '%s' in class '%s'." + .formatted(field.getName(), configClass.getCanonicalName()); + throw new RuntimeException(message); } writer.println(INDENT + "# " + fieldDoc.help()); String fieldPrefix = ""; @@ -99,28 +91,28 @@ private void generateConnectorYaml(Class configClass, PrintWriter writer) { writer.flush(); } - private void generateConnectorYaml(Class connectorClass, Connector connectorDef, PrintWriter writer) { + private void generateConnectorYamlFile(Class connectorClass, Connector connectorDef, PrintWriter writer) { log.info("Processing connector definition : {}", connectorDef); writer.println("# " + connectorDef.type() + " connector : " + connectorClass.getName()); writer.println(); writer.println("# " + connectorDef.help()); writer.println(); - generateConnectorYaml(connectorDef.configClass(), writer); + generateConnectorYamlFile(connectorDef.configClass(), writer); } - private void generatorConnectorYamls(String outputDir) throws IOException { + private void generatorConnectorYamlFiles(String outputDir) throws IOException { Set> connectorClasses = reflections.getTypesAnnotatedWith(Connector.class); log.info("Retrieve all `Connector` annotated classes : {}", connectorClasses); for (Class connectorClass : connectorClasses) { - Connector connectorDef = connectorClass.getDeclaredAnnotation(Connector.class); - try (FileOutputStream fos = new FileOutputStream( - Paths.get( - outputDir, - "pulsar-io-" + connectorDef.name() - + "-" + connectorDef.type().name().toLowerCase()).toString() + ".yml")) { + final Connector connectorDef = connectorClass.getDeclaredAnnotation(Connector.class); + final String name = connectorDef.name().toLowerCase(); + final String type = connectorDef.type().name().toLowerCase(); + final String filename = "pulsar-io-%s-%s.yml".formatted(name, type); + final Path outputPath = Path.of(outputDir, filename); + try (FileOutputStream fos = new FileOutputStream(outputPath.toFile())) { PrintWriter pw = new PrintWriter(new OutputStreamWriter(fos, StandardCharsets.UTF_8)); - generateConnectorYaml(connectorClass, connectorDef, pw); + generateConnectorYamlFile(connectorClass, connectorDef, pw); pw.flush(); } } @@ -130,23 +122,14 @@ private void generatorConnectorYamls(String outputDir) throws IOException { * Args for stats generator. */ private static class MainArgs { - @Parameter( - names = { - "-o", "--output-dir" - }, - description = "The output dir to dump connector docs", - required = true - ) + names = {"-o", "--output-dir"}, + description = "The output dir to dump connector docs", + required = true) String outputDir = null; - @Parameter( - names = { - "-h", "--help" - }, - description = "Show this help message") + @Parameter(names = {"-h", "--help"}, description = "Show this help message") boolean help = false; - } public static void main(String[] args) throws Exception { @@ -169,7 +152,7 @@ public static void main(String[] args) throws Exception { } ConnectorDocGenerator docGen = new ConnectorDocGenerator(); - docGen.generatorConnectorYamls(mainArgs.outputDir); + docGen.generatorConnectorYamlFiles(mainArgs.outputDir); } } diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java index a8c7358bf9415..9f42dbda7be1b 100644 --- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java @@ -242,6 +242,11 @@ public class ElasticSearchConfig implements Serializable { ) private String primaryFields = ""; + @FieldDoc( + required = false, + defaultValue = "", + help = "The SSL config for elastic search." + ) private ElasticSearchSslConfig ssl = new ElasticSearchSslConfig(); @FieldDoc( From add2594c0fa9c472232338ea94d9467fd9bb6df3 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Thu, 13 Apr 2023 08:51:52 +0200 Subject: [PATCH 040/494] [improve][ci]Add branch-3.0 to owasp checks (#20081) (cherry picked from commit 4d8156fca0582a350d30557d9243c851c4a45830) --- .github/workflows/ci-owasp-dependency-check.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index dba1c92ca28dd..8aec501f8d868 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -39,6 +39,7 @@ jobs: matrix: include: - branch: master + - branch: branch-3.0 - branch: branch-2.11 - branch: branch-2.10 jdk: 11 From d7f97dd6a7b6cf8f063c3c9a17bc6e58eaca1caa Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Mon, 17 Apr 2023 19:32:13 +0800 Subject: [PATCH 041/494] [improve] [broker] Close temporary open ledger in BookkeeperBucketSnapshotStorage (#20111) (cherry picked from commit b50e8802a5224dd68832e263e7046650771a1a4e) --- .../BookkeeperBucketSnapshotStorage.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index e7d4f9301dd36..9c30ccf1c0b7e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -66,22 +66,41 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - return openLedger(bucketId).thenCompose( - ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0). - thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture snapshotFuture = + getLedgerEntry(ledgerHandle, 0, 0) + .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())); + + snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return snapshotFuture; + }); } @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { - return openLedger(bucketId).thenCompose( - ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, - lastSegmentEntryId).thenApply(this::parseSnapshotSegmentEntries)); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture> parseFuture = + getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) + .thenApply(this::parseSnapshotSegmentEntries); + + parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return parseFuture; + }); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return openLedger(bucketId).thenApply(LedgerHandle::getLength); + return openLedger(bucketId).thenCompose(ledgerHandle -> { + CompletableFuture lengthFuture = + CompletableFuture.completedFuture(ledgerHandle.getLength()); + + lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); + + return lengthFuture; + }); } @Override From dc339df94abf89e8f8513c37adeb261931489813 Mon Sep 17 00:00:00 2001 From: Andrey Yegorov <8622884+dlg99@users.noreply.github.com> Date: Tue, 18 Apr 2023 14:37:41 -0700 Subject: [PATCH 042/494] [fix][io] KCA: handle kafka sources that use commitRecord (#20121) (cherry picked from commit 46a65fdc182a34753ebabfa35f63c0f6c765462f) --- .../io/kafka/connect/KafkaConnectSource.java | 43 ++++++- .../connect/ErrRecFileStreamSourceTask.java | 33 +++++ .../connect/KafkaConnectSourceErrRecTest.java | 118 ++++++++++++++++++ 3 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java create mode 100644 pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java diff --git a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java index f5f6efd08bd99..f2ee8a8e6cafe 100644 --- a/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java +++ b/pulsar-io/kafka-connect-adaptor/src/main/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSource.java @@ -27,6 +27,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.connect.json.JsonConverterConfig; import org.apache.kafka.connect.source.SourceRecord; import org.apache.pulsar.client.api.Schema; @@ -57,6 +59,7 @@ public void open(Map config, SourceContext sourceContext) throws config.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, false); } log.info("jsonWithEnvelope: {}", jsonWithEnvelope); + super.open(config, sourceContext); } @@ -69,17 +72,26 @@ public synchronized KafkaSourceRecord processSourceRecord(final SourceRecord src private static final AvroData avroData = new AvroData(1000); - private class KafkaSourceRecord extends AbstractKafkaSourceRecord> + public class KafkaSourceRecord extends AbstractKafkaSourceRecord> implements KVRecord { + final int keySize; + final int valueSize; + + final SourceRecord srcRecord; + KafkaSourceRecord(SourceRecord srcRecord) { super(srcRecord); + this.srcRecord = srcRecord; + byte[] keyBytes = keyConverter.fromConnectData( srcRecord.topic(), srcRecord.keySchema(), srcRecord.key()); + keySize = keyBytes != null ? keyBytes.length : 0; this.key = keyBytes != null ? Optional.of(Base64.getEncoder().encodeToString(keyBytes)) : Optional.empty(); byte[] valueBytes = valueConverter.fromConnectData( srcRecord.topic(), srcRecord.valueSchema(), srcRecord.value()); + valueSize = valueBytes != null ? valueBytes.length : 0; this.value = new KeyValue<>(keyBytes, valueBytes); @@ -145,6 +157,35 @@ public KeyValueEncodingType getKeyValueEncodingType() { } } + @Override + public void ack() { + // first try to commitRecord() for the current record in the batch + // then call super.ack() which calls commit() after complete batch of records is processed + try { + if (log.isDebugEnabled()) { + log.debug("commitRecord() for record: {}", srcRecord); + } + getSourceTask().commitRecord(srcRecord, + new RecordMetadata( + new TopicPartition(srcRecord.topic() == null + ? topicName.orElse("UNDEFINED") + : srcRecord.topic(), + srcRecord.kafkaPartition() == null ? 0 : srcRecord.kafkaPartition()), + -1L, // baseOffset == -1L means no offset + 0, // batchIndex, doesn't matter if baseOffset == -1L + null == srcRecord.timestamp() ? -1L : srcRecord.timestamp(), + keySize, // serializedKeySize + valueSize // serializedValueSize + )); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("Source task failed to commit record, " + + "source task should resend data, will get duplicate", e); + return; + } + super.ack(); + } + } } diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java new file mode 100644 index 0000000000000..cbdd4c41bf692 --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/ErrRecFileStreamSourceTask.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.io.kafka.connect; + +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.connect.file.FileStreamSourceTask; +import org.apache.kafka.connect.source.SourceRecord; + +public class ErrRecFileStreamSourceTask extends FileStreamSourceTask { + + @Override + public void commitRecord(SourceRecord record, RecordMetadata metadata) throws InterruptedException { + throw new org.apache.kafka.connect.errors.ConnectException("blah"); + } + +} diff --git a/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java new file mode 100644 index 0000000000000..9872e1fbc7e2f --- /dev/null +++ b/pulsar-io/kafka-connect-adaptor/src/test/java/org/apache/pulsar/io/kafka/connect/KafkaConnectSourceErrRecTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.kafka.connect; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.connect.file.FileStreamSourceConnector; +import org.apache.kafka.connect.runtime.TaskConfig; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.schema.KeyValue; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.SourceContext; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import java.io.File; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +/** + * Test the implementation of {@link KafkaConnectSource}. + */ +@Slf4j +public class KafkaConnectSourceErrRecTest extends ProducerConsumerBase { + + private Map config = new HashMap<>(); + private String offsetTopicName; + // The topic to publish data to, for kafkaSource + private String topicName; + private KafkaConnectSource kafkaConnectSource; + private File tempFile; + private SourceContext context; + private PulsarClient client; + + @BeforeMethod + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + + config.put(TaskConfig.TASK_CLASS_CONFIG, "org.apache.pulsar.io.kafka.connect.ErrRecFileStreamSourceTask"); + config.put(PulsarKafkaWorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); + config.put(PulsarKafkaWorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); + + this.offsetTopicName = "persistent://my-property/my-ns/kafka-connect-source-offset"; + config.put(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG, offsetTopicName); + + this.topicName = "persistent://my-property/my-ns/kafka-connect-source"; + config.put(FileStreamSourceConnector.TOPIC_CONFIG, topicName); + tempFile = File.createTempFile("some-file-name", null); + config.put(FileStreamSourceConnector.FILE_CONFIG, tempFile.getAbsoluteFile().toString()); + config.put(FileStreamSourceConnector.TASK_BATCH_SIZE_CONFIG, String.valueOf(FileStreamSourceConnector.DEFAULT_TASK_BATCH_SIZE)); + + this.context = mock(SourceContext.class); + this.client = PulsarClient.builder() + .serviceUrl(brokerUrl.toString()) + .build(); + when(context.getPulsarClient()).thenReturn(this.client); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + if (this.client != null) { + this.client.close(); + } + tempFile.delete(); + super.internalCleanup(); + } + + @Test + public void testCommitRecordCalled() throws Exception { + kafkaConnectSource = new KafkaConnectSource(); + kafkaConnectSource.open(config, context); + + // use FileStreamSourceConnector, each line is a record, need "\n" and end of each record. + OutputStream os = Files.newOutputStream(tempFile.toPath()); + + String line1 = "This is the first line\n"; + os.write(line1.getBytes()); + os.flush(); + os.close(); + + Record> record = kafkaConnectSource.read(); + + assertTrue(record instanceof KafkaConnectSource.KafkaSourceRecord); + + try { + record.ack(); + fail("expected exception"); + } catch (Exception e) { + log.info("got exception", e); + assertTrue(e instanceof org.apache.kafka.connect.errors.ConnectException); + } + } +} From 88e05b5b9ee65d245276dfdca87e9200206fede6 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 19 Apr 2023 14:45:29 +0800 Subject: [PATCH 043/494] [fix][sec] spring.version=5.3.27 to fix CVE-2023-20863 (#20124) Signed-off-by: tison (cherry picked from commit 866d405726bb6527db954201f2260620181747dc) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e7e00db74e4db..252cedbf65927 100644 --- a/pom.xml +++ b/pom.xml @@ -238,7 +238,7 @@ flexible messaging model and an intuitive client API. 1.6.0 1.0 9.1.6 - 5.3.26 + 5.3.27 4.5.13 4.4.15 0.5.11 From d08c3cbc70468011430bd7482431d34014c90ed5 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 19 Apr 2023 16:44:12 +0800 Subject: [PATCH 044/494] [improve][broker] Make timer execute immediately after load index (#20126) (cherry picked from commit 9b723022436cc1a150af765103e8d343679f92ce) --- .../delayed/bucket/BucketDelayedDeliveryTracker.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index f57248acbb7bd..b4d1745e22f3a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -541,7 +541,7 @@ public long getBufferMemoryUsage() { @Override public synchronized NavigableSet getScheduledMessages(int maxMessages) { - if (!checkPendingOpDone()) { + if (!checkPendingLoadDone()) { if (log.isDebugEnabled()) { log.debug("[{}] Skip getScheduledMessages to wait for bucket snapshot load finish.", dispatcher.getName()); @@ -628,11 +628,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa if (timeout != null) { timeout.cancel(); } - timeout = timer.newTimeout(this, tickTimeMillis, TimeUnit.MILLISECONDS); + timeout = timer.newTimeout(this, 0, TimeUnit.MILLISECONDS); } }); - if (!checkPendingOpDone() || loadFuture.isCompletedExceptionally()) { + if (!checkPendingLoadDone() || loadFuture.isCompletedExceptionally()) { break; } } @@ -651,7 +651,7 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa return positions; } - private synchronized boolean checkPendingOpDone() { + private synchronized boolean checkPendingLoadDone() { if (pendingLoad == null || pendingLoad.isDone()) { pendingLoad = null; return true; From 4c5f9d2acf94fab01206258ffbb1068bd9afd97e Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Wed, 19 Apr 2023 23:40:28 +0800 Subject: [PATCH 045/494] [fix][broker] Fix tenant admin authorization bug. (#20068) Co-authored-by: druidliu (cherry picked from commit fc17c1d98a3c1edd975c131d174a9ef69887d9cd) --- .../authorization/AuthorizationService.java | 27 +++++-------------- .../pulsar/broker/auth/AuthorizationTest.java | 22 ++++++++------- .../AuthorizationProducerConsumerTest.java | 16 +++++++++++ .../proxy/ProxyAuthorizationTest.java | 14 +++++++--- 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index 0c61219b57a50..a9225f5e48fa9 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -28,6 +28,7 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; @@ -182,13 +183,7 @@ public CompletableFuture canProduceAsync(TopicName topicName, String ro if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { - if (isSuperUser) { - return CompletableFuture.completedFuture(true); - } else { - return provider.canProduceAsync(topicName, role, authenticationData); - } - }); + return provider.allowTopicOperationAsync(topicName, role, TopicOperation.PRODUCE, authenticationData); } /** @@ -207,13 +202,9 @@ public CompletableFuture canConsumeAsync(TopicName topicName, String ro if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { - if (isSuperUser) { - return CompletableFuture.completedFuture(true); - } else { - return provider.canConsumeAsync(topicName, role, authenticationData, subscription); - } - }); + + return provider.allowTopicOperationAsync(topicName, role, TopicOperation.CONSUME, + new AuthenticationDataSubscription(authenticationData, subscription)); } public boolean canProduce(TopicName topicName, String role, AuthenticationDataSource authenticationData) @@ -289,13 +280,7 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { - if (isSuperUser) { - return CompletableFuture.completedFuture(true); - } else { - return provider.canLookupAsync(topicName, role, authenticationData); - } - }); + return provider.allowTopicOperationAsync(topicName, role, TopicOperation.LOOKUP, authenticationData); } public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index 58cf4ee418ea4..4fce7c50e1c44 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -79,10 +79,13 @@ public void cleanup() throws Exception { public void simple() throws Exception { AuthorizationService auth = pulsar.getBrokerService().getAuthorizationService(); - assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - + try { + assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); + fail("Should throw exception when tenant not exist"); + } catch (Exception ignored) {} admin.clusters().createCluster("c1", ClusterData.builder().build()); - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); + String tenantAdmin = "role1"; + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1"))); waitForChange(); admin.namespaces().createNamespace("p1/c1/ns1"); waitForChange(); @@ -215,21 +218,22 @@ public void simple() throws Exception { SubscriptionAuthMode.Prefix); waitForChange(); - assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null)); + assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null)); - try { - assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "sub1")); - fail(); - } catch (Exception ignored) {} + // tenant admin can consume all topics, even if SubscriptionAuthMode.Prefix mode + assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1")); try { assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "sub2")); fail(); } catch (Exception ignored) {} - assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "role1-sub1")); + assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "role1-sub1")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "role2-sub2")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "pulsar.super_user", null, "role3-sub1")); + // tenant admin can produce all topics + assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); + admin.namespaces().deleteNamespace("p1/c1/ns1"); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 0ce3b7df07d1f..dd351286d2e5e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -1035,6 +1035,22 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol return CompletableFuture.completedFuture(grantRoles.contains(role)); } + @Override + public CompletableFuture allowTopicOperationAsync(TopicName topic, String role, + TopicOperation operation, + AuthenticationDataSource authData) { + switch (operation) { + + case PRODUCE: + return canProduceAsync(topic, role, authData); + case CONSUME: + return canConsumeAsync(topic, role, authData, authData.getSubscription()); + case LOOKUP: + return canLookupAsync(topic, role, authData); + } + return super.allowTopicOperationAsync(topic, role, operation, authData); + } + @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authData) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java index a3b26a4a9d122..29327d3d4ebe1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Sets; import java.util.EnumSet; import java.util.Optional; @@ -83,10 +84,13 @@ protected void cleanup() throws Exception { public void test() throws Exception { AuthorizationService auth = service.getAuthorizationService(); - assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - + try { + assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); + fail("Should throw exception when tenant not exist"); + } catch (Exception ignored) {} admin.clusters().createCluster(configClusterName, ClusterData.builder().build()); - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); + String tenantAdmin = "role1"; + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1"))); waitForChange(); admin.namespaces().createNamespace("p1/c1/ns1"); waitForChange(); @@ -117,6 +121,10 @@ public void test() throws Exception { assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null, null)); + // tenant admin can produce/consume all topics, even if SubscriptionAuthMode.Prefix mode + assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1")); + assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); + admin.namespaces().deleteNamespace("p1/c1/ns1"); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); From 17b59a23244d6b00031cddd5ee33ba8aac2df6f4 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 20 Apr 2023 08:21:08 +0800 Subject: [PATCH 046/494] [improve][broker] Optimize delayed metadata index bitmap (#20136) (cherry picked from commit 6dc0b0ea38ab9dc410a24b19fd7567b9e013837d) --- .../bucket/BucketDelayedDeliveryTracker.java | 3 ++ .../delayed/bucket/ImmutableBucket.java | 53 +++++++++++-------- .../broker/delayed/bucket/MutableBucket.java | 11 ++-- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index b4d1745e22f3a..b17387e276e2b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -485,6 +485,9 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List> asyncLoadNextBucketSnapshotEntry(b this.setLastSegmentEntryId(metadataList.size()); this.recoverDelayedIndexBitMapAndNumber(nextSnapshotEntryIndex, metadataList); List firstScheduleTimestamps = metadataList.stream().map( - SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); + SnapshotSegmentMetadata::getMinScheduleTimestamp).toList(); this.setFirstScheduleTimestamps(firstScheduleTimestamps); return nextSnapshotEntryIndex + 1; @@ -139,25 +138,37 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b }); } + /** + * Recover delayed index bit map and message numbers. + * @throws InvalidRoaringFormat invalid bitmap serialization format + */ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, - List segmentMetadata) { - this.delayedIndexBitMap.clear(); - MutableLong numberMessages = new MutableLong(0); - for (int i = startSnapshotIndex; i < segmentMetadata.size(); i++) { - Map bitByteStringMap = segmentMetadata.get(i).getDelayedIndexBitMapMap(); - bitByteStringMap.forEach((leaderId, bitSetString) -> { - boolean exist = this.delayedIndexBitMap.containsKey(leaderId); - RoaringBitmap bitSet = - new ImmutableRoaringBitmap(bitSetString.asReadOnlyByteBuffer()).toRoaringBitmap(); - numberMessages.add(bitSet.getCardinality()); - if (!exist) { - this.delayedIndexBitMap.put(leaderId, bitSet); - } else { - this.delayedIndexBitMap.get(leaderId).or(bitSet); + List segmentMetaList) { + delayedIndexBitMap.clear(); // cleanup dirty bm + final var numberMessages = new MutableLong(0); + for (int i = startSnapshotIndex; i < segmentMetaList.size(); i++) { + for (final var entry : segmentMetaList.get(i).getDelayedIndexBitMapMap().entrySet()) { + final var ledgerId = entry.getKey(); + final var bs = entry.getValue(); + final var sbm = new RoaringBitmap(); + try { + sbm.deserialize(bs.asReadOnlyByteBuffer()); + } catch (IOException e) { + throw new InvalidRoaringFormat(e.getMessage()); } - }); + numberMessages.add(sbm.getCardinality()); + delayedIndexBitMap.compute(ledgerId, (lId, bm) -> { + if (bm == null) { + return sbm; + } + bm.or(sbm); + return bm; + }); + } } - this.setNumberBucketDelayedMessages(numberMessages.getValue()); + // optimize bm + delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize); + setNumberBucketDelayedMessages(numberMessages.getValue()); } CompletableFuture> getRemainSnapshotSegment() { @@ -193,7 +204,7 @@ CompletableFuture asyncDeleteBucketSnapshot(BucketDelayedMessageIndexStats stats.recordFailEvent(BucketDelayedMessageIndexStats.Type.delete); } else { log.info("[{}] Delete bucket snapshot finish, bucketId: {}, bucketKey: {}", - dispatcherName, bucketId, bucketKey); + dispatcherName, bucketId, bucketKey); stats.recordSuccessEvent(BucketDelayedMessageIndexStats.Type.delete, System.currentTimeMillis() - deleteStartTime); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index e49ebe9606e01..f404d5d02c15a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -116,10 +116,13 @@ Pair createImmutableBucketAndAsyncPersistent( Iterator> iterator = bitMap.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - byte[] array = new byte[entry.getValue().serializedSizeInBytes()]; - entry.getValue().serialize(ByteBuffer.wrap(array)); - segmentMetadataBuilder.putDelayedIndexBitMap(entry.getKey(), ByteString.copyFrom(array)); + final var entry = iterator.next(); + final var lId = entry.getKey(); + final var bm = entry.getValue(); + bm.runOptimize(); + final var array = new byte[bm.serializedSizeInBytes()]; + bm.serialize(ByteBuffer.wrap(array)); + segmentMetadataBuilder.putDelayedIndexBitMap(lId, ByteString.copyFrom(array)); iterator.remove(); } From 3c4a1ce026db0602cda1d3c65ce9211e9e88f02b Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Thu, 20 Apr 2023 09:26:30 +0800 Subject: [PATCH 047/494] [improve][broker] Cache LedgerHandle in BookkeeperBucketSnapshotStorage (#20117) (cherry picked from commit d3fa998aa7c0a7a9452079ef2ff05bccf6b273cf) --- .../BookkeeperBucketSnapshotStorage.java | 52 +++++++++---------- .../BookkeeperBucketSnapshotStorageTest.java | 43 +++++++++++++++ 2 files changed, 69 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 9c30ccf1c0b7e..18a4c322f7b27 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import javax.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; @@ -48,6 +49,8 @@ public class BookkeeperBucketSnapshotStorage implements BucketSnapshotStorage { private final ServiceConfiguration config; private BookKeeper bookKeeper; + private final Map> ledgerHandleFutureCache = new ConcurrentHashMap<>(); + public BookkeeperBucketSnapshotStorage(PulsarService pulsar) { this.pulsar = pulsar; this.config = pulsar.getConfig(); @@ -66,45 +69,30 @@ public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMet @Override public CompletableFuture getBucketSnapshotMetadata(long bucketId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture snapshotFuture = - getLedgerEntry(ledgerHandle, 0, 0) - .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement())); - - snapshotFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return snapshotFuture; - }); + return getLedgerHandle(bucketId).thenCompose(ledgerHandle -> getLedgerEntry(ledgerHandle, 0, 0) + .thenApply(entryEnumeration -> parseSnapshotMetadataEntry(entryEnumeration.nextElement()))); } @Override public CompletableFuture> getBucketSnapshotSegment(long bucketId, long firstSegmentEntryId, long lastSegmentEntryId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture> parseFuture = - getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) - .thenApply(this::parseSnapshotSegmentEntries); - - parseFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return parseFuture; - }); + return getLedgerHandle(bucketId).thenCompose( + ledgerHandle -> getLedgerEntry(ledgerHandle, firstSegmentEntryId, lastSegmentEntryId) + .thenApply(this::parseSnapshotSegmentEntries)); } @Override public CompletableFuture getBucketSnapshotLength(long bucketId) { - return openLedger(bucketId).thenCompose(ledgerHandle -> { - CompletableFuture lengthFuture = - CompletableFuture.completedFuture(ledgerHandle.getLength()); - - lengthFuture.whenComplete((__, e) -> closeLedger(ledgerHandle)); - - return lengthFuture; - }); + return getLedgerHandle(bucketId).thenCompose( + ledgerHandle -> CompletableFuture.completedFuture(ledgerHandle.getLength())); } @Override public CompletableFuture deleteBucketSnapshot(long bucketId) { + CompletableFuture ledgerHandleFuture = ledgerHandleFutureCache.remove(bucketId); + if (ledgerHandleFuture != null) { + ledgerHandleFuture.whenComplete((lh, ex) -> closeLedger(lh)); + } return deleteLedger(bucketId); } @@ -178,6 +166,18 @@ private CompletableFuture createLedger(String bucketKey, String to return future; } + private CompletableFuture getLedgerHandle(Long ledgerId) { + CompletableFuture ledgerHandleCompletableFuture = + ledgerHandleFutureCache.computeIfAbsent(ledgerId, k -> openLedger(ledgerId)); + // remove future of completed exceptionally + ledgerHandleCompletableFuture.whenComplete((__, ex) -> { + if (ex != null) { + ledgerHandleFutureCache.remove(ledgerId, ledgerHandleCompletableFuture); + } + }); + return ledgerHandleCompletableFuture; + } + private CompletableFuture openLedger(Long ledgerId) { final CompletableFuture future = new CompletableFuture<>(); bookKeeper.asyncOpenLedger( diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index a628b58e10d32..7cb6b8d5865bb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -204,4 +205,46 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted Assert.assertTrue(bucketSnapshotLength > 0L); } + @Test + public void testConcurrencyGet() throws ExecutionException, InterruptedException { + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + .setMinScheduleTimestamp(System.currentTimeMillis()) + .setMaxScheduleTimestamp(System.currentTimeMillis()) + .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); + + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = + DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + .addMetadataList(segmentMetadata) + .build(); + List bucketSnapshotSegments = new ArrayList<>(); + + long timeMillis = System.currentTimeMillis(); + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = + DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis).build(); + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + bucketSnapshotSegments.add(snapshotSegment); + bucketSnapshotSegments.add(snapshotSegment); + + CompletableFuture future = + bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, + bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); + Long bucketId = future.get(); + Assert.assertNotNull(bucketId); + + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + CompletableFuture future0 = CompletableFuture.runAsync(() -> { + List list = + bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join(); + Assert.assertTrue(list.size() > 0); + }); + futures.add(future0); + } + + FutureUtil.waitForAll(futures).join(); + } + } From 800a82a3a9f0d480b594cf8268410ada00b4d313 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 20 Apr 2023 00:17:02 -0500 Subject: [PATCH 048/494] [fix][broker] Producer/Consumer should call allowTopicOperationAsync (#20142) Fixes: https://github.com/apache/pulsar/issues/20066 ### Motivation In https://github.com/apache/pulsar/pull/20068 we changed the way that the `AuthorizationService` is implemented. I think we should change the `Consumer` and the `Producer` logic to call the correct `AuthorizationService` method. Given that our goal is to deprecate the `AuthorizationService` methods for `canProduce` and `canConsume`, this change helps us move in the right direction. ### Modifications * Update `Producer` and `Consumer` in broker to call the `AuthorizationService#allowTopicOperationAsync` method. ### Verifying this change This change is trivial. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: Skipping PR as I ran tests locally. (cherry picked from commit dc5e497b2a42910146690c6b4e922e0a80bf1587) --- .../java/org/apache/pulsar/broker/service/Consumer.java | 8 ++++++-- .../java/org/apache/pulsar/broker/service/Producer.java | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 1ee3f513ef288..a3f9da41e6b35 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -43,6 +43,7 @@ import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageId; @@ -56,6 +57,7 @@ import org.apache.pulsar.common.api.proto.MessageIdData; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.schema.SchemaType; @@ -901,8 +903,10 @@ public String toString() { public CompletableFuture checkPermissionsAsync() { TopicName topicName = TopicName.get(subscription.getTopicName()); if (cnx.getBrokerService().getAuthorizationService() != null) { - return cnx.getBrokerService().getAuthorizationService().canConsumeAsync(topicName, appId, - cnx.getAuthenticationData(), subscription.getName()) + AuthenticationDataSubscription authData = + new AuthenticationDataSubscription(cnx.getAuthenticationData(), subscription.getName()); + return cnx.getBrokerService().getAuthorizationService() + .allowTopicOperationAsync(topicName, TopicOperation.CONSUME, appId, authData) .handle((ok, e) -> { if (e != null) { log.warn("[{}] Get unexpected error while authorizing [{}] {}", appId, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java index 5b62e3261e64f..53b79f06e8e24 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Producer.java @@ -51,6 +51,7 @@ import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData.ClusterUrl; +import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.policies.data.stats.NonPersistentPublisherStatsImpl; import org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl; import org.apache.pulsar.common.protocol.Commands; @@ -781,7 +782,7 @@ public CompletableFuture checkPermissionsAsync() { TopicName topicName = TopicName.get(topic.getName()); if (cnx.getBrokerService().getAuthorizationService() != null) { return cnx.getBrokerService().getAuthorizationService() - .canProduceAsync(topicName, appId, cnx.getAuthenticationData()) + .allowTopicOperationAsync(topicName, TopicOperation.PRODUCE, appId, cnx.getAuthenticationData()) .handle((ok, ex) -> { if (ex != null) { log.warn("[{}] Get unexpected error while autorizing [{}] {}", appId, topic.getName(), From 38360d06ba16fbf6339d325ae200d603850b47ec Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 20 Apr 2023 00:17:22 -0500 Subject: [PATCH 049/494] [revert] "[fix][broker] Fix tenant admin authorization bug. (#20068)" (#20143) This reverts commit fc17c1d98a3c1edd975c131d174a9ef69887d9cd. ### Motivation In https://github.com/apache/pulsar/pull/20068 we changed the way that the `AuthorizationService` is implemented. I think this approach could have unintended consequences. Instead, I think we should change the `Consumer` and the `Producer` logic to call the correct `AuthorizationService` method. I propose an update to the `Consumer` and `Producer` logic here #20142. Given that our goal is to deprecate the `AuthorizationService` methods for `canProduce` and `canConsume`, I think we should not change their implementations. ### Modifications * Revert https://github.com/apache/pulsar/pull/20068 ### Verifying this change This change is trivial. It removes certain test changes that were only made to make the previous PR work. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: Skipping PR as I ran tests locally. (cherry picked from commit 00dc7a0691b496065dba6af0c6de64af4973886e) --- .../authorization/AuthorizationService.java | 27 ++++++++++++++----- .../pulsar/broker/auth/AuthorizationTest.java | 22 +++++++-------- .../AuthorizationProducerConsumerTest.java | 16 ----------- .../proxy/ProxyAuthorizationTest.java | 14 +++------- 4 files changed, 33 insertions(+), 46 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java index a9225f5e48fa9..0c61219b57a50 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/AuthorizationService.java @@ -28,7 +28,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; -import org.apache.pulsar.broker.authentication.AuthenticationDataSubscription; import org.apache.pulsar.broker.authentication.AuthenticationParameters; import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.common.naming.NamespaceName; @@ -183,7 +182,13 @@ public CompletableFuture canProduceAsync(TopicName topicName, String ro if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.allowTopicOperationAsync(topicName, role, TopicOperation.PRODUCE, authenticationData); + return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { + if (isSuperUser) { + return CompletableFuture.completedFuture(true); + } else { + return provider.canProduceAsync(topicName, role, authenticationData); + } + }); } /** @@ -202,9 +207,13 @@ public CompletableFuture canConsumeAsync(TopicName topicName, String ro if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - - return provider.allowTopicOperationAsync(topicName, role, TopicOperation.CONSUME, - new AuthenticationDataSubscription(authenticationData, subscription)); + return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { + if (isSuperUser) { + return CompletableFuture.completedFuture(true); + } else { + return provider.canConsumeAsync(topicName, role, authenticationData, subscription); + } + }); } public boolean canProduce(TopicName topicName, String role, AuthenticationDataSource authenticationData) @@ -280,7 +289,13 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol if (!this.conf.isAuthorizationEnabled()) { return CompletableFuture.completedFuture(true); } - return provider.allowTopicOperationAsync(topicName, role, TopicOperation.LOOKUP, authenticationData); + return provider.isSuperUser(role, authenticationData, conf).thenComposeAsync(isSuperUser -> { + if (isSuperUser) { + return CompletableFuture.completedFuture(true); + } else { + return provider.canLookupAsync(topicName, role, authenticationData); + } + }); } public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java index 4fce7c50e1c44..58cf4ee418ea4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthorizationTest.java @@ -79,13 +79,10 @@ public void cleanup() throws Exception { public void simple() throws Exception { AuthorizationService auth = pulsar.getBrokerService().getAuthorizationService(); - try { - assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - fail("Should throw exception when tenant not exist"); - } catch (Exception ignored) {} + assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); + admin.clusters().createCluster("c1", ClusterData.builder().build()); - String tenantAdmin = "role1"; - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1"))); + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); waitForChange(); admin.namespaces().createNamespace("p1/c1/ns1"); waitForChange(); @@ -218,22 +215,21 @@ public void simple() throws Exception { SubscriptionAuthMode.Prefix); waitForChange(); - assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); + assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null)); assertTrue(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null)); - // tenant admin can consume all topics, even if SubscriptionAuthMode.Prefix mode - assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1")); + try { + assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "sub1")); + fail(); + } catch (Exception ignored) {} try { assertFalse(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "sub2")); fail(); } catch (Exception ignored) {} - assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "role1-sub1")); + assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role1", null, "role1-sub1")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "role2", null, "role2-sub2")); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "pulsar.super_user", null, "role3-sub1")); - // tenant admin can produce all topics - assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); - admin.namespaces().deleteNamespace("p1/c1/ns1"); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index dd351286d2e5e..0ce3b7df07d1f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -1035,22 +1035,6 @@ public CompletableFuture canLookupAsync(TopicName topicName, String rol return CompletableFuture.completedFuture(grantRoles.contains(role)); } - @Override - public CompletableFuture allowTopicOperationAsync(TopicName topic, String role, - TopicOperation operation, - AuthenticationDataSource authData) { - switch (operation) { - - case PRODUCE: - return canProduceAsync(topic, role, authData); - case CONSUME: - return canConsumeAsync(topic, role, authData, authData.getSubscription()); - case LOOKUP: - return canLookupAsync(topic, role, authData); - } - return super.allowTopicOperationAsync(topic, role, operation, authData); - } - @Override public CompletableFuture grantPermissionAsync(NamespaceName namespace, Set actions, String role, String authData) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java index 29327d3d4ebe1..a3b26a4a9d122 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/websocket/proxy/ProxyAuthorizationTest.java @@ -25,7 +25,6 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; import com.google.common.collect.Sets; import java.util.EnumSet; import java.util.Optional; @@ -84,13 +83,10 @@ protected void cleanup() throws Exception { public void test() throws Exception { AuthorizationService auth = service.getAuthorizationService(); - try { - assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); - fail("Should throw exception when tenant not exist"); - } catch (Exception ignored) {} + assertFalse(auth.canLookup(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); + admin.clusters().createCluster(configClusterName, ClusterData.builder().build()); - String tenantAdmin = "role1"; - admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet(tenantAdmin), Sets.newHashSet("c1"))); + admin.tenants().createTenant("p1", new TenantInfoImpl(Sets.newHashSet("role1"), Sets.newHashSet("c1"))); waitForChange(); admin.namespaces().createNamespace("p1/c1/ns1"); waitForChange(); @@ -121,10 +117,6 @@ public void test() throws Exception { assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null)); assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), "my-role", null, null)); - // tenant admin can produce/consume all topics, even if SubscriptionAuthMode.Prefix mode - assertTrue(auth.canConsume(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null, "sub1")); - assertTrue(auth.canProduce(TopicName.get("persistent://p1/c1/ns1/ds1"), tenantAdmin, null)); - admin.namespaces().deleteNamespace("p1/c1/ns1"); admin.tenants().deleteTenant("p1"); admin.clusters().deleteCluster("c1"); From bafae5d3f48c6c858e48f3cfc7e0c7f212ece2f4 Mon Sep 17 00:00:00 2001 From: ran Date: Thu, 20 Apr 2023 16:18:06 +0800 Subject: [PATCH 050/494] [fix][broker] Fix entry filter feature for the non-persistent topic (#20141) (cherry picked from commit 575cf2331dd3ac0048923487c6c7904eda4301e6) --- .../nonpersistent/NonPersistentTopic.java | 5 ++- .../service/plugin/FilterEntryTest.java | 15 +++++-- .../broker/stats/SubscriptionStatsTest.java | 42 ++++++++++--------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 317b8df6b9a82..33258b06726b5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -26,7 +26,7 @@ import io.netty.buffer.ByteBuf; import io.netty.util.concurrent.FastThreadLocal; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -199,7 +199,8 @@ public void publishMessage(ByteBuf data, PublishContext callback) { // entry internally retains data so, duplicateBuffer should be release here duplicateBuffer.release(); if (subscription.getDispatcher() != null) { - subscription.getDispatcher().sendMessages(Collections.singletonList(entry)); + // Dispatcher needs to call the set method to support entry filter feature. + subscription.getDispatcher().sendMessages(Arrays.asList(entry)); } else { // it happens when subscription is created but dispatcher is not created as consumer is not added // yet diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java index 4b9d91fbde219..b868858646c50 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/plugin/FilterEntryTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.EntryFilterSupport; +import org.apache.pulsar.broker.service.Subscription; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -286,10 +287,16 @@ public void testFilter() throws Exception { } + @DataProvider(name = "topicProvider") + public Object[][] topicProvider() { + return new Object[][]{ + {"persistent://prop/ns-abc/topic" + UUID.randomUUID()}, + {"non-persistent://prop/ns-abc/topic" + UUID.randomUUID()}, + }; + } - @Test - public void testFilteredMsgCount() throws Throwable { - String topic = "persistent://prop/ns-abc/topic" + UUID.randomUUID(); + @Test(dataProvider = "topicProvider") + public void testFilteredMsgCount(String topic) throws Throwable { String subName = "sub"; try (Producer producer = pulsarClient.newProducer(Schema.STRING) @@ -298,7 +305,7 @@ public void testFilteredMsgCount() throws Throwable { .subscriptionName(subName).subscribe()) { // mock entry filters - PersistentSubscription subscription = (PersistentSubscription) pulsar.getBrokerService() + Subscription subscription = pulsar.getBrokerService() .getTopicReference(topic).get().getSubscription(subName); Dispatcher dispatcher = subscription.getDispatcher(); Field field = EntryFilterSupport.class.getDeclaredField("entryFilters"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java index bf9c1d540bf87..d5e0066a86f15 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/SubscriptionStatsTest.java @@ -211,21 +211,22 @@ public void testSubscriptionStats(final String topic, final String subName, bool hasFilterField.set(dispatcher, true); } - for (int i = 0; i < 100; i++) { - producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send(); - } - for (int i = 0; i < 100; i++) { + int rejectedCount = 100; + int acceptCount = 100; + int scheduleCount = 100; + for (int i = 0; i < rejectedCount; i++) { producer.newMessage().property("REJECT", " ").value(UUID.randomUUID().toString()).send(); } - for (int i = 0; i < 100; i++) { + for (int i = 0; i < acceptCount; i++) { + producer.newMessage().property("ACCEPT", " ").value(UUID.randomUUID().toString()).send(); + } + for (int i = 0; i < scheduleCount; i++) { producer.newMessage().property("RESCHEDULE", " ").value(UUID.randomUUID().toString()).send(); } - for (;;) { - Message message = consumer.receive(10, TimeUnit.SECONDS); - if (message == null) { - break; - } + for (int i = 0; i < acceptCount; i++) { + Message message = consumer.receive(1, TimeUnit.SECONDS); + Assert.assertNotNull(message); consumer.acknowledge(message); } @@ -263,12 +264,12 @@ public void testSubscriptionStats(final String topic, final String subName, bool .mapToDouble(m-> m.value).sum(); if (setFilter) { - Assert.assertEquals(filterAccepted, 100); - if (isPersistent) { - Assert.assertEquals(filterRejected, 100); - // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount - Assert.assertEquals(throughFilter, filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter); - } + Assert.assertEquals(filterAccepted, acceptCount); + Assert.assertEquals(filterRejected, rejectedCount); + // Only works on the test, if there are some markers, + // the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount + Assert.assertEquals(throughFilter, + filterAccepted + filterRejected + filterRescheduled, 0.01 * throughFilter); } else { Assert.assertEquals(throughFilter, 0D); Assert.assertEquals(filterAccepted, 0D); @@ -282,19 +283,20 @@ public void testSubscriptionStats(final String topic, final String subName, bool Assert.assertEquals(rescheduledMetrics.size(), 0); } - testSubscriptionStatsAdminApi(topic, subName, setFilter); + testSubscriptionStatsAdminApi(topic, subName, setFilter, acceptCount, rejectedCount); } - private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter) throws Exception { + private void testSubscriptionStatsAdminApi(String topic, String subName, boolean setFilter, + int acceptCount, int rejectedCount) throws Exception { boolean persistent = TopicName.get(topic).isPersistent(); TopicStats topicStats = admin.topics().getStats(topic); SubscriptionStats stats = topicStats.getSubscriptions().get(subName); Assert.assertNotNull(stats); if (setFilter) { - Assert.assertEquals(stats.getFilterAcceptedMsgCount(), 100); + Assert.assertEquals(stats.getFilterAcceptedMsgCount(), acceptCount); if (persistent) { - Assert.assertEquals(stats.getFilterRejectedMsgCount(), 100); + Assert.assertEquals(stats.getFilterRejectedMsgCount(), rejectedCount); // Only works on the test, if there are some markers, the filterProcessCount will be not equal with rejectedCount + rescheduledCount + acceptCount Assert.assertEquals(stats.getFilterProcessedMsgCount(), stats.getFilterAcceptedMsgCount() + stats.getFilterRejectedMsgCount() From d5edcc449b4e8511697d16151721ada218c90a3e Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Thu, 20 Apr 2023 18:11:52 +0800 Subject: [PATCH 051/494] [fix][txn] Fix transaction is not aborted when send or ACK failed (#20055) (cherry picked from commit 00d09cbbd2b3063fecbdc3988b5eef7824f40ce0) --- .../transaction/TransactionProduceTest.java | 35 ++++++++-- .../impl/transaction/TransactionImpl.java | 70 +++++++++---------- 2 files changed, 62 insertions(+), 43 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java index cdbb1563280b4..ddd8cf0790321 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionProduceTest.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import lombok.Cleanup; @@ -43,6 +45,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.transaction.Transaction; @@ -51,10 +54,11 @@ import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException; import org.awaitility.Awaitility; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; /** @@ -70,7 +74,7 @@ public class TransactionProduceTest extends TransactionTestBase { private static final String ACK_COMMIT_TOPIC = NAMESPACE1 + "/ack-commit"; private static final String ACK_ABORT_TOPIC = NAMESPACE1 + "/ack-abort"; private static final int NUM_PARTITIONS = 16; - @BeforeMethod + @BeforeClass protected void setup() throws Exception { setUpBase(1, NUM_PARTITIONS, PRODUCE_COMMIT_TOPIC, TOPIC_PARTITION); admin.topics().createPartitionedTopic(PRODUCE_ABORT_TOPIC, TOPIC_PARTITION); @@ -78,7 +82,7 @@ protected void setup() throws Exception { admin.topics().createPartitionedTopic(ACK_ABORT_TOPIC, TOPIC_PARTITION); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) protected void cleanup() throws Exception { super.internalCleanup(); } @@ -369,5 +373,26 @@ private int getPendingAckCount(String topic, String subscriptionName) throws Exc return pendingAckCount; } - + @Test + public void testCommitFailure() throws Exception { + Transaction txn = pulsarClient.newTransaction().build().get(); + final String topic = NAMESPACE1 + "/test-commit-failure"; + @Cleanup + final Producer producer = pulsarClient.newProducer().topic(topic).create(); + producer.newMessage(txn).value(new byte[1024 * 1024 * 10]).sendAsync(); + try { + txn.commit().get(); + Assert.fail(); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof PulsarClientException.TransactionHasOperationFailedException); + Assert.assertEquals(txn.getState(), Transaction.State.ABORTED); + } + try { + getPulsarServiceList().get(0).getTransactionMetadataStoreService().getTxnMeta(txn.getTxnID()) + .getNow(null); + Assert.fail(); + } catch (CompletionException e) { + Assert.assertTrue(e.getCause() instanceof CoordinatorException.TransactionNotFoundException); + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java index b7e085ed82a85..d1260ba045e6d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionImpl.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import io.netty.util.Timeout; import io.netty.util.TimerTask; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -186,13 +187,14 @@ public void registerAckOp(CompletableFuture newAckFuture) { @Override public CompletableFuture commit() { timeout.cancel(); - return checkIfOpenOrCommitting().thenCompose((value) -> { + return checkState(State.OPEN, State.COMMITTING).thenCompose((value) -> { CompletableFuture commitFuture = new CompletableFuture<>(); this.state = State.COMMITTING; opFuture.whenComplete((v, e) -> { if (hasOpsFailed) { - abort().whenComplete((vx, ex) -> commitFuture.completeExceptionally(new PulsarClientException - .TransactionHasOperationFailedException())); + checkState(State.COMMITTING).thenCompose(__ -> internalAbort()).whenComplete((vx, ex) -> + commitFuture.completeExceptionally( + new PulsarClientException.TransactionHasOperationFailedException())); } else { tcClient.commitAsync(txnId) .whenComplete((vx, ex) -> { @@ -216,28 +218,30 @@ public CompletableFuture commit() { @Override public CompletableFuture abort() { timeout.cancel(); - return checkIfOpenOrAborting().thenCompose(value -> { - CompletableFuture abortFuture = new CompletableFuture<>(); - this.state = State.ABORTING; - opFuture.whenComplete((v, e) -> { - tcClient.abortAsync(txnId).whenComplete((vx, ex) -> { + return checkState(State.OPEN, State.ABORTING).thenCompose(__ -> internalAbort()); + } - if (ex != null) { - if (ex instanceof TransactionNotFoundException - || ex instanceof InvalidTxnStatusException) { - this.state = State.ERROR; - } - abortFuture.completeExceptionally(ex); - } else { - this.state = State.ABORTED; - abortFuture.complete(null); + private CompletableFuture internalAbort() { + CompletableFuture abortFuture = new CompletableFuture<>(); + this.state = State.ABORTING; + opFuture.whenComplete((v, e) -> { + tcClient.abortAsync(txnId).whenComplete((vx, ex) -> { + + if (ex != null) { + if (ex instanceof TransactionNotFoundException + || ex instanceof InvalidTxnStatusException) { + this.state = State.ERROR; } + abortFuture.completeExceptionally(ex); + } else { + this.state = State.ABORTED; + abortFuture.complete(null); + } - }); }); - - return abortFuture; }); + + return abortFuture; } @Override @@ -261,25 +265,15 @@ public boolean checkIfOpen(CompletableFuture completableFuture) { } } - private CompletableFuture checkIfOpenOrCommitting() { - if (state == State.OPEN || state == State.COMMITTING) { - return CompletableFuture.completedFuture(null); - } else { - return invalidTxnStatusFuture(); - } - } - - private CompletableFuture checkIfOpenOrAborting() { - if (state == State.OPEN || state == State.ABORTING) { - return CompletableFuture.completedFuture(null); - } else { - return invalidTxnStatusFuture(); + private CompletableFuture checkState(State... expectedStates) { + final State actualState = STATE_UPDATE.get(this); + for (State expectedState : expectedStates) { + if (actualState == expectedState) { + return CompletableFuture.completedFuture(null); + } } - } - - private CompletableFuture invalidTxnStatusFuture() { return FutureUtil.failedFuture(new InvalidTxnStatusException("[" + txnIdMostBits + ":" - + txnIdLeastBits + "] with unexpected state : " - + state.name() + ", expect " + State.OPEN + " state!")); + + txnIdLeastBits + "] with unexpected state: " + actualState.name() + ", expect: " + + Arrays.toString(expectedStates))); } } From 0743c774fd6621a8ddcdf50b7f28dc8f68a6e492 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 21 Apr 2023 09:51:55 +0800 Subject: [PATCH 052/494] [improve][broker] Optimization protobuf code in the bucket delayed tracker (#20158) (cherry picked from commit a20d5e96acc91f8099845e8924e84cb0b5632f3f) --- pulsar-broker/pom.xml | 2 + .../BookkeeperBucketSnapshotStorage.java | 24 ++--- .../pulsar/broker/delayed/bucket/Bucket.java | 7 +- .../bucket/BucketDelayedDeliveryTracker.java | 13 ++- .../delayed/bucket/BucketSnapshotStorage.java | 4 +- .../CombinedSegmentDelayedIndexQueue.java | 22 ++++- .../delayed/bucket/DelayedIndexQueue.java | 12 ++- .../delayed/bucket/ImmutableBucket.java | 19 ++-- .../broker/delayed/bucket/MutableBucket.java | 29 +++--- .../TripleLongPriorityDelayedIndexQueue.java | 25 +++-- ...> DelayedMessageIndexBucketMetadata.proto} | 11 +-- .../DelayedMessageIndexBucketSegment.proto | 35 +++++++ .../BookkeeperBucketSnapshotStorageTest.java | 95 +++++++++---------- .../delayed/MockBucketSnapshotStorage.java | 14 +-- .../delayed/bucket/DelayedIndexQueueTest.java | 70 ++++++-------- 15 files changed, 210 insertions(+), 172 deletions(-) rename pulsar-broker/src/main/proto/{DelayedMessageIndexBucketSnapshotFormat.proto => DelayedMessageIndexBucketMetadata.proto} (85%) create mode 100644 pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 31b335e1aea68..25b31b4f2b7bd 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -566,6 +566,7 @@ **/ResourceUsage.proto **/TransactionPendingAck.proto + **/DelayedMessageIndexBucketSegment.proto @@ -610,6 +611,7 @@ ${project.basedir}/src/main/proto/TransactionPendingAck.proto ${project.basedir}/src/main/proto/ResourceUsage.proto + ${project.basedir}/src/main/proto/DelayedMessageIndexBucketSegment.proto generated-sources/lightproto/java generated-sources/lightproto/java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 18a4c322f7b27..040bbbc586f49 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import com.google.protobuf.InvalidProtocolBufferException; -import java.io.IOException; +import io.netty.buffer.ByteBuf; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -36,8 +36,8 @@ import org.apache.bookkeeper.mledger.impl.LedgerMetadataUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -126,7 +126,8 @@ private CompletableFuture addSnapshotSegments(LedgerHandle ledgerHandle, private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { try { - return SnapshotMetadata.parseFrom(ledgerEntry.getEntry()); + ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); + return SnapshotMetadata.parseFrom(entryBuffer.nioBuffer()); } catch (InvalidProtocolBufferException e) { throw new BucketSnapshotSerializationException(e); } @@ -134,15 +135,14 @@ private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { private List parseSnapshotSegmentEntries(Enumeration entryEnumeration) { List snapshotMetadataList = new ArrayList<>(); - try { - while (entryEnumeration.hasMoreElements()) { - LedgerEntry ledgerEntry = entryEnumeration.nextElement(); - snapshotMetadataList.add(SnapshotSegment.parseFrom(ledgerEntry.getEntry())); - } - return snapshotMetadataList; - } catch (IOException e) { - throw new BucketSnapshotSerializationException(e); + while (entryEnumeration.hasMoreElements()) { + LedgerEntry ledgerEntry = entryEnumeration.nextElement(); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); + snapshotSegment.parseFrom(entryBuffer, entryBuffer.readableBytes()); + snapshotMetadataList.add(snapshotSegment); } + return snapshotMetadataList; } @NotNull diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java index 4d7d3aa512be6..a1693b1553d97 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/Bucket.java @@ -31,7 +31,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.RoaringBitmap; @@ -132,8 +133,8 @@ long getAndUpdateBucketId() { } CompletableFuture asyncSaveBucketSnapshot( - ImmutableBucket bucket, DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata, - List bucketSnapshotSegments) { + ImmutableBucket bucket, SnapshotMetadata snapshotMetadata, + List bucketSnapshotSegments) { final String bucketKey = bucket.bucketKey(); final String cursorName = Codec.decode(cursor.getName()); final String topicName = dispatcherName.substring(0, dispatcherName.lastIndexOf(" / " + cursorName)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index b17387e276e2b..c90064c9137bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -53,8 +53,8 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.delayed.AbstractDelayedDeliveryTracker; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.apache.pulsar.common.util.FutureUtil; @@ -286,8 +286,7 @@ private void afterCreateImmutableBucket(Pair immu // Put indexes back into the shared queue and downgrade to memory mode synchronized (BucketDelayedDeliveryTracker.this) { immutableBucket.getSnapshotSegments().ifPresent(snapshotSegments -> { - for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment : - snapshotSegments) { + for (SnapshotSegment snapshotSegment : snapshotSegments) { for (DelayedIndex delayedIndex : snapshotSegment.getIndexesList()) { sharedBucketPriorityQueue.add(delayedIndex.getTimestamp(), delayedIndex.getLedgerId(), delayedIndex.getEntryId()); @@ -450,7 +449,7 @@ private synchronized CompletableFuture asyncMergeBucketSnapshot(List>> getRemainFutures = + List>> getRemainFutures = buckets.stream().map(ImmutableBucket::getRemainSnapshotSegment).toList(); return FutureUtil.waitForAll(getRemainFutures) @@ -601,11 +600,11 @@ public synchronized NavigableSet getScheduledMessages(int maxMessa bucket.asyncDeleteBucketSnapshot(stats); return; } - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex + DelayedIndex lastDelayedIndex = indexList.get(indexList.size() - 1); this.snapshotSegmentLastIndexTable.put(lastDelayedIndex.getLedgerId(), lastDelayedIndex.getEntryId(), bucket); - for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : indexList) { + for (DelayedIndex index : indexList) { sharedBucketPriorityQueue.add(index.getTimestamp(), index.getLedgerId(), index.getEntryId()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java index 51c89bed47af2..7464ef9cd3f63 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketSnapshotStorage.java @@ -20,8 +20,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; public interface BucketSnapshotStorage { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java index 5655a26878296..006938e9ed271 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/CombinedSegmentDelayedIndexQueue.java @@ -24,8 +24,8 @@ import java.util.PriorityQueue; import javax.annotation.concurrent.NotThreadSafe; import lombok.AllArgsConstructor; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; @NotThreadSafe class CombinedSegmentDelayedIndexQueue implements DelayedIndexQueue { @@ -40,8 +40,8 @@ static class Node { } private static final Comparator COMPARATOR_NODE = (node1, node2) -> DelayedIndexQueue.COMPARATOR.compare( - node1.segmentList.get(node1.segmentListCursor).getIndexes(node1.segmentCursor), - node2.segmentList.get(node2.segmentListCursor).getIndexes(node2.segmentCursor)); + node1.segmentList.get(node1.segmentListCursor).getIndexeAt(node1.segmentCursor), + node2.segmentList.get(node2.segmentListCursor).getIndexeAt(node2.segmentCursor)); private final PriorityQueue kpq; @@ -77,7 +77,7 @@ private DelayedIndex getValue(boolean needAdvanceCursor) { Objects.requireNonNull(node); SnapshotSegment snapshotSegment = node.segmentList.get(node.segmentListCursor); - DelayedIndex delayedIndex = snapshotSegment.getIndexes(node.segmentCursor); + DelayedIndex delayedIndex = snapshotSegment.getIndexeAt(node.segmentCursor); if (!needAdvanceCursor) { return delayedIndex; } @@ -104,4 +104,16 @@ private DelayedIndex getValue(boolean needAdvanceCursor) { return delayedIndex; } + + @Override + public void popToObject(DelayedIndex delayedIndex) { + DelayedIndex value = getValue(true); + delayedIndex.copyFrom(value); + } + + @Override + public long peekTimestamp() { + DelayedIndex value = getValue(false); + return value.getTimestamp(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java index dee476c376ec9..f1209a3137a45 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueue.java @@ -20,10 +20,10 @@ import java.util.Comparator; import java.util.Objects; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; interface DelayedIndexQueue { - Comparator COMPARATOR = (o1, o2) -> { + Comparator COMPARATOR = (o1, o2) -> { if (!Objects.equals(o1.getTimestamp(), o2.getTimestamp())) { return Long.compare(o1.getTimestamp(), o2.getTimestamp()); } else if (!Objects.equals(o1.getLedgerId(), o2.getLedgerId())) { @@ -35,7 +35,11 @@ interface DelayedIndexQueue { boolean isEmpty(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek(); + DelayedIndex peek(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop(); + DelayedIndex pop(); + + void popToObject(DelayedIndex delayedIndex); + + long peekTimestamp(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java index 57de5c84fcd82..0932f51f350ce 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/ImmutableBucket.java @@ -32,9 +32,9 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.mutable.MutableLong; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.roaringbitmap.InvalidRoaringFormat; import org.roaringbitmap.RoaringBitmap; @@ -43,7 +43,7 @@ class ImmutableBucket extends Bucket { @Setter - private List snapshotSegments; + private List snapshotSegments; boolean merging = false; @@ -55,7 +55,7 @@ class ImmutableBucket extends Bucket { super(dispatcherName, cursor, sequencer, storage, startLedgerId, endLedgerId); } - public Optional> getSnapshotSegments() { + public Optional> getSnapshotSegments() { return Optional.ofNullable(snapshotSegments); } @@ -84,7 +84,7 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b } }), BucketSnapshotPersistenceException.class, MaxRetryTimes) .thenApply(snapshotMetadata -> { - List metadataList = + List metadataList = snapshotMetadata.getMetadataListList(); // Skip all already reach schedule time snapshot segments @@ -125,10 +125,9 @@ private CompletableFuture> asyncLoadNextBucketSnapshotEntry(b return Collections.emptyList(); } - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = + SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - List indexList = - snapshotSegment.getIndexesList(); + List indexList = snapshotSegment.getIndexesList(); this.setCurrentSegmentEntryId(nextSegmentEntryId); if (isRecover) { this.asyncUpdateSnapshotLength(); @@ -171,7 +170,7 @@ private void recoverDelayedIndexBitMapAndNumber(int startSnapshotIndex, setNumberBucketDelayedMessages(numberMessages.getValue()); } - CompletableFuture> getRemainSnapshotSegment() { + CompletableFuture> getRemainSnapshotSegment() { int nextSegmentEntryId = currentSegmentEntryId + 1; if (nextSegmentEntryId > lastSegmentEntryId) { return CompletableFuture.completedFuture(Collections.emptyList()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index f404d5d02c15a..b7e9e68f1bdc7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -30,10 +30,10 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.commons.lang3.tuple.Pair; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.roaringbitmap.RoaringBitmap; @@ -75,14 +75,16 @@ Pair createImmutableBucketAndAsyncPersistent( List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); Map bitMap = new HashMap<>(); - SnapshotSegment.Builder snapshotSegmentBuilder = SnapshotSegment.newBuilder(); + SnapshotSegment snapshotSegment = new SnapshotSegment(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); List firstScheduleTimestamps = new ArrayList<>(); long currentTimestampUpperLimit = 0; long currentFirstTimestamp = 0L; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex delayedIndex = delayedIndexQueue.peek(); + DelayedIndex delayedIndex = snapshotSegment.addIndexe(); + delayedIndexQueue.popToObject(delayedIndex); + long timestamp = delayedIndex.getTimestamp(); if (currentTimestampUpperLimit == 0) { currentFirstTimestamp = timestamp; @@ -100,16 +102,13 @@ Pair createImmutableBucketAndAsyncPersistent( sharedQueue.add(timestamp, ledgerId, entryId); } - delayedIndexQueue.pop(); numMessages++; bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); - snapshotSegmentBuilder.addIndexes(delayedIndex); - - if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peek().getTimestamp() > currentTimestampUpperLimit + if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peekTimestamp() > currentTimestampUpperLimit || (maxIndexesPerBucketSnapshotSegment != -1 - && snapshotSegmentBuilder.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { + && snapshotSegment.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { segmentMetadataBuilder.setMaxScheduleTimestamp(timestamp); segmentMetadataBuilder.setMinScheduleTimestamp(currentFirstTimestamp); currentTimestampUpperLimit = 0; @@ -129,8 +128,8 @@ Pair createImmutableBucketAndAsyncPersistent( segmentMetadataList.add(segmentMetadataBuilder.build()); segmentMetadataBuilder.clear(); - bucketSnapshotSegments.add(snapshotSegmentBuilder.build()); - snapshotSegmentBuilder.clear(); + bucketSnapshotSegments.add(snapshotSegment); + snapshotSegment = new SnapshotSegment(); } } @@ -153,8 +152,8 @@ Pair createImmutableBucketAndAsyncPersistent( // Add the first snapshot segment last message to snapshotSegmentLastMessageTable checkArgument(!bucketSnapshotSegments.isEmpty()); - SnapshotSegment snapshotSegment = bucketSnapshotSegments.get(0); - DelayedIndex lastDelayedIndex = snapshotSegment.getIndexes(snapshotSegment.getIndexesCount() - 1); + SnapshotSegment firstSnapshotSegment = bucketSnapshotSegments.get(0); + DelayedIndex lastDelayedIndex = firstSnapshotSegment.getIndexeAt(firstSnapshotSegment.getIndexesCount() - 1); Pair result = Pair.of(bucket, lastDelayedIndex); CompletableFuture future = asyncSaveBucketSnapshot(bucket, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java index b8d54bd78b428..4faee3b17f17f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/TripleLongPriorityDelayedIndexQueue.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import javax.annotation.concurrent.NotThreadSafe; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; @NotThreadSafe @@ -41,17 +41,28 @@ public boolean isEmpty() { } @Override - public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek() { - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setTimestamp(queue.peekN1()) - .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()).build(); + public DelayedIndex peek() { + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(queue.peekN1()) + .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()); return delayedIndex; } @Override - public DelayedMessageIndexBucketSnapshotFormat.DelayedIndex pop() { - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex peek = peek(); + public DelayedIndex pop() { + DelayedIndex peek = peek(); queue.pop(); return peek; } + + @Override + public void popToObject(DelayedIndex delayedIndex) { + delayedIndex.setTimestamp(queue.peekN1()) + .setLedgerId(queue.peekN2()).setEntryId(queue.peekN3()); + queue.pop(); + } + + @Override + public long peekTimestamp() { + return queue.peekN1(); + } } diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto similarity index 85% rename from pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto rename to pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto index 6996b860c5249..01b770c567d09 100644 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSnapshotFormat.proto +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketMetadata.proto @@ -21,12 +21,7 @@ syntax = "proto2"; package pulsar.delay; option java_package = "org.apache.pulsar.broker.delayed.proto"; option optimize_for = SPEED; - -message DelayedIndex { - required uint64 timestamp = 1; - required uint64 ledger_id = 2; - required uint64 entry_id = 3; -} +option java_multiple_files = true; message SnapshotSegmentMetadata { map delayed_index_bit_map = 1; @@ -34,10 +29,6 @@ message SnapshotSegmentMetadata { required uint64 min_schedule_timestamp = 3; } -message SnapshotSegment { - repeated DelayedIndex indexes = 1; -} - message SnapshotMetadata { repeated SnapshotSegmentMetadata metadata_list = 1; } diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto new file mode 100644 index 0000000000000..633d6a8f1615c --- /dev/null +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +syntax = "proto2"; + +package pulsar.delay; +option java_package = "org.apache.pulsar.broker.delayed.proto"; +option optimize_for = SPEED; +option java_multiple_files = true; + +message DelayedIndex { + required uint64 timestamp = 1; + required uint64 ledger_id = 2; + required uint64 entry_id = 3; +} + +message SnapshotSegment { + repeated DelayedIndex indexes = 1; + map delayed_index_bit_map = 2; +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java index 7cb6b8d5865bb..d26f38fa2bc2b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/BookkeeperBucketSnapshotStorageTest.java @@ -29,7 +29,10 @@ import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.delayed.bucket.BookkeeperBucketSnapshotStorage; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegmentMetadata; import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -60,9 +63,8 @@ protected void cleanup() throws Exception { @Test public void testCreateSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build(); - List bucketSnapshotSegments = new ArrayList<>(); + SnapshotMetadata snapshotMetadata = SnapshotMetadata.newBuilder().build(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); @@ -72,24 +74,23 @@ public void testCreateSnapshot() throws ExecutionException, InterruptedException @Test public void testGetSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L) + .setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -99,13 +100,13 @@ public void testGetSnapshot() throws ExecutionException, InterruptedException { Long bucketId = future.get(); Assert.assertNotNull(bucketId); - CompletableFuture> bucketSnapshotSegment = + CompletableFuture> bucketSnapshotSegment = bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3); - List snapshotSegments = bucketSnapshotSegment.get(); + List snapshotSegments = bucketSnapshotSegment.get(); Assert.assertEquals(2, snapshotSegments.size()); - for (DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment segment : snapshotSegments) { - for (DelayedMessageIndexBucketSnapshotFormat.DelayedIndex index : segment.getIndexesList()) { + for (SnapshotSegment segment : snapshotSegments) { + for (DelayedIndex index : segment.getIndexesList()) { Assert.assertEquals(100L, index.getLedgerId()); Assert.assertEquals(10L, index.getEntryId()); Assert.assertEquals(timeMillis, index.getTimestamp()); @@ -121,17 +122,17 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce map.put(100L, ByteString.copyFrom("test1", StandardCharsets.UTF_8)); map.put(200L, ByteString.copyFrom("test2", StandardCharsets.UTF_8)); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMaxScheduleTimestamp(timeMillis) .setMinScheduleTimestamp(timeMillis) .putAllDelayedIndexBitMap(map).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, @@ -139,10 +140,10 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce Long bucketId = future.get(); Assert.assertNotNull(bucketId); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata bucketSnapshotMetadata = + SnapshotMetadata bucketSnapshotMetadata = bucketSnapshotStorage.getBucketSnapshotMetadata(bucketId).get(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata metadata = + SnapshotSegmentMetadata metadata = bucketSnapshotMetadata.getMetadataList(0); Assert.assertEquals(timeMillis, metadata.getMaxScheduleTimestamp()); @@ -152,9 +153,9 @@ public void testGetSnapshotMetadata() throws ExecutionException, InterruptedExce @Test public void testDeleteSnapshot() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder().build(); - List bucketSnapshotSegments = new ArrayList<>(); + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder().build(); + List bucketSnapshotSegments = new ArrayList<>(); CompletableFuture future = bucketSnapshotStorage.createBucketSnapshot(snapshotMetadata, bucketSnapshotSegments, UUID.randomUUID().toString(), TOPIC_NAME, CURSOR_NAME); @@ -173,24 +174,22 @@ public void testDeleteSnapshot() throws ExecutionException, InterruptedException @Test public void testGetBucketSnapshotLength() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -207,24 +206,22 @@ public void testGetBucketSnapshotLength() throws ExecutionException, Interrupted @Test public void testConcurrencyGet() throws ExecutionException, InterruptedException { - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata segmentMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegmentMetadata.newBuilder() + SnapshotSegmentMetadata segmentMetadata = + SnapshotSegmentMetadata.newBuilder() .setMinScheduleTimestamp(System.currentTimeMillis()) .setMaxScheduleTimestamp(System.currentTimeMillis()) .putDelayedIndexBitMap(100L, ByteString.copyFrom(new byte[1])).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata snapshotMetadata = - DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata.newBuilder() + SnapshotMetadata snapshotMetadata = + SnapshotMetadata.newBuilder() .addMetadataList(segmentMetadata) .build(); - List bucketSnapshotSegments = new ArrayList<>(); + List bucketSnapshotSegments = new ArrayList<>(); long timeMillis = System.currentTimeMillis(); - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex delayedIndex = - DelayedMessageIndexBucketSnapshotFormat.DelayedIndex.newBuilder().setLedgerId(100L).setEntryId(10L) - .setTimestamp(timeMillis).build(); - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment snapshotSegment = - DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment.newBuilder().addIndexes(delayedIndex).build(); + DelayedIndex delayedIndex = new DelayedIndex().setLedgerId(100L).setEntryId(10L).setTimestamp(timeMillis); + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.addIndexe().copyFrom(delayedIndex); bucketSnapshotSegments.add(snapshotSegment); bucketSnapshotSegments.add(snapshotSegment); @@ -237,7 +234,7 @@ public void testConcurrencyGet() throws ExecutionException, InterruptedException List> futures = new ArrayList<>(); for (int i = 0; i < 100; i++) { CompletableFuture future0 = CompletableFuture.runAsync(() -> { - List list = + List list = bucketSnapshotStorage.getBucketSnapshotSegment(bucketId, 1, 3).join(); Assert.assertTrue(list.size() > 0); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java index 9e924bdeda341..dc1c0e09ca276 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockBucketSnapshotStorage.java @@ -36,8 +36,8 @@ import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.delayed.bucket.BucketSnapshotStorage; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotMetadata; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -139,13 +139,9 @@ public CompletableFuture> getBucketSnapshotSegment(long bu long lastEntryId = Math.min(lastSegmentEntryId, this.bucketSnapshots.get(bucketId).size()); for (int i = (int) firstSegmentEntryId; i <= lastEntryId ; i++) { ByteBuf byteBuf = this.bucketSnapshots.get(bucketId).get(i); - SnapshotSegment snapshotSegment; - try { - snapshotSegment = SnapshotSegment.parseFrom(byteBuf.nioBuffer()); - snapshotSegments.add(snapshotSegment); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } + SnapshotSegment snapshotSegment = new SnapshotSegment(); + snapshotSegment.parseFrom(byteBuf, byteBuf.readableBytes()); + snapshotSegments.add(snapshotSegment); } return snapshotSegments; }, executorService); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java index 8f87f0d49a20c..5dc3dcc7cb9a7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/bucket/DelayedIndexQueueTest.java @@ -23,8 +23,8 @@ import java.util.List; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.DelayedIndex; -import org.apache.pulsar.broker.delayed.proto.DelayedMessageIndexBucketSnapshotFormat.SnapshotSegment; +import org.apache.pulsar.broker.delayed.proto.DelayedIndex; +import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; import org.apache.pulsar.common.util.collections.TripleLongPriorityQueue; import org.testng.Assert; import org.testng.annotations.Test; @@ -35,27 +35,19 @@ public class DelayedIndexQueueTest { @Test public void testCompare() { DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(2).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(2).setLedgerId(2L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(1).setLedgerId(2L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); - delayedIndex = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(1L) - .build(); - delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(1).setLedgerId(1L).setEntryId(2L) - .build(); + delayedIndex = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(1L); + delayedIndex2 = new DelayedIndex().setTimestamp(1).setLedgerId(1L).setEntryId(2L); Assert.assertTrue(COMPARATOR.compare(delayedIndex, delayedIndex2) < 0); } @@ -63,21 +55,19 @@ public void testCompare() { public void testCombinedSegmentDelayedIndexQueue() { List listA = new ArrayList<>(); for (int i = 0; i < 10; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L); listA.add(delayedIndex); } - SnapshotSegment snapshotSegmentA1 = SnapshotSegment.newBuilder().addAllIndexes(listA).build(); + SnapshotSegment snapshotSegmentA1 = new SnapshotSegment(); + snapshotSegmentA1.addAllIndexes(listA); List listA2 = new ArrayList<>(); for (int i = 10; i < 20; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(1L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(1L).setEntryId(1L); listA2.add(delayedIndex); } - SnapshotSegment snapshotSegmentA2 = SnapshotSegment.newBuilder().addAllIndexes(listA2).build(); + SnapshotSegment snapshotSegmentA2 = new SnapshotSegment(); + snapshotSegmentA2.addAllIndexes(listA2); List segmentListA = new ArrayList<>(); segmentListA.add(snapshotSegmentA1); @@ -85,36 +75,32 @@ public void testCombinedSegmentDelayedIndexQueue() { List listB = new ArrayList<>(); for (int i = 0; i < 9; i++) { - DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) - .build(); + DelayedIndex delayedIndex = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L); - DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) - .build(); + DelayedIndex delayedIndex2 = new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L); listB.add(delayedIndex); listB.add(delayedIndex2); } - SnapshotSegment snapshotSegmentB = SnapshotSegment.newBuilder().addAllIndexes(listB).build(); + SnapshotSegment snapshotSegmentB = new SnapshotSegment(); + snapshotSegmentB.addAllIndexes(listB); List segmentListB = new ArrayList<>(); segmentListB.add(snapshotSegmentB); - segmentListB.add(SnapshotSegment.newBuilder().build()); + segmentListB.add(new SnapshotSegment()); List listC = new ArrayList<>(); for (int i = 10; i < 30; i+=2) { DelayedIndex delayedIndex = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(1L) - .build(); + new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(1L); DelayedIndex delayedIndex2 = - DelayedIndex.newBuilder().setTimestamp(i).setLedgerId(2L).setEntryId(2L) - .build(); + new DelayedIndex().setTimestamp(i).setLedgerId(2L).setEntryId(2L); listC.add(delayedIndex); listC.add(delayedIndex2); } - SnapshotSegment snapshotSegmentC = SnapshotSegment.newBuilder().addAllIndexes(listC).build(); + SnapshotSegment snapshotSegmentC = new SnapshotSegment(); + snapshotSegmentC.addAllIndexes(listC); List segmentListC = new ArrayList<>(); segmentListC.add(snapshotSegmentC); @@ -123,11 +109,14 @@ public void testCombinedSegmentDelayedIndexQueue() { int count = 0; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex pop = delayedIndexQueue.pop(); + DelayedIndex pop = new DelayedIndex(); + delayedIndexQueue.popToObject(pop); log.info("{} , {}, {}", pop.getTimestamp(), pop.getLedgerId(), pop.getEntryId()); count++; if (!delayedIndexQueue.isEmpty()) { DelayedIndex peek = delayedIndexQueue.peek(); + long timestamp = delayedIndexQueue.peekTimestamp(); + Assert.assertEquals(timestamp, peek.getTimestamp()); Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } @@ -147,10 +136,13 @@ public void TripleLongPriorityDelayedIndexQueueTest() { int count = 0; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex pop = delayedIndexQueue.pop(); + DelayedIndex pop = new DelayedIndex(); + delayedIndexQueue.popToObject(pop); count++; if (!delayedIndexQueue.isEmpty()) { DelayedIndex peek = delayedIndexQueue.peek(); + long timestamp = delayedIndexQueue.peekTimestamp(); + Assert.assertEquals(timestamp, peek.getTimestamp()); Assert.assertTrue(COMPARATOR.compare(peek, pop) >= 0); } } From 044dd84966fe1faad725568456fbab26a75eebe8 Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Fri, 21 Apr 2023 09:57:06 +0800 Subject: [PATCH 053/494] [improve][schema] Add admin cli for testCompatibility (#19974) ### Motivation 1. add admin cli `testCompatibility` 2. add `admin.schemas().testCompatibility()` test ### Modifications 1. add admin cli `testCompatibility` 2. add `admin.schemas().testCompatibility()` test ### Verifying this change add test for it - [ ] Make sure that the change passes the CI checks. *(Please pick either of the following options)* This change is a trivial rework / code cleanup without any test coverage. *(or)* This change is already covered by existing tests, such as *(please describe tests)*. *(or)* This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Does this pull request potentially affect one of the following parts: *If the box was checked, please highlight the changes* - [ ] Dependencies (add or upgrade a dependency) - [ ] The public API - [ ] The schema - [ ] The default values of configurations - [ ] The threading model - [ ] The binary protocol - [ ] The REST endpoints - [x] The admin CLI options - [ ] The metrics - [ ] Anything that affects deployment ### Documentation - [ ] `doc` - [ ] `doc-required` - [x] `doc-not-needed` - [ ] `doc-complete` ### Matching PR in forked repository PR in forked repository: https://github.com/congbobo184/pulsar/pull/14 (cherry picked from commit fce6e737a7ff785859117b216cbb12f999ddeb94) --- .../broker/admin/AdminApiSchemaTest.java | 29 +++++++++++++++++++ .../schema/IsCompatibilityResponse.java | 1 - .../pulsar/admin/cli/PulsarAdminToolTest.java | 5 ++++ .../apache/pulsar/admin/cli/CmdSchemas.java | 17 +++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java index b37114f180216..f67bd6fcfce5b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaTest.java @@ -49,6 +49,8 @@ import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy; import org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.schema.IsCompatibilityResponse; +import org.apache.pulsar.common.protocol.schema.PostSchemaPayload; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaInfoWithVersion; import org.apache.pulsar.common.schema.SchemaType; @@ -438,4 +440,31 @@ public void testGetSchemaCompatibilityStrategyWhenSetBrokerLevelAndSchemaAutoUpd admin.namespaces().getSchemaCompatibilityStrategy(schemaCompatibilityNamespace), SchemaCompatibilityStrategy.UNDEFINED)); } + + @Test + public void testCompatibility() throws Exception { + String topicName = schemaCompatibilityNamespace + "/testCompatibility"; + try { + admin.schemas().getSchemaInfo(topicName); + fail(); + } catch (PulsarAdminException.NotFoundException e) { + assertEquals(e.getMessage(), "Schema not found"); + } + Map properties = new HashMap<>(); + PostSchemaPayload postSchemaPayload = new PostSchemaPayload("STRING", "", properties); + admin.schemas().createSchema(topicName, postSchemaPayload); + IsCompatibilityResponse isCompatibilityResponse = + admin.schemas().testCompatibility(topicName, postSchemaPayload); + + assertTrue(isCompatibilityResponse.isCompatibility()); + assertEquals(isCompatibilityResponse.getSchemaCompatibilityStrategy(), SchemaCompatibilityStrategy.FULL.name()); + postSchemaPayload = new PostSchemaPayload("INT8", "", properties); + try { + admin.schemas().testCompatibility(topicName, postSchemaPayload); + fail(); + } catch (Exception e) { + assertTrue(e instanceof PulsarAdminException.ServerSideErrorException); + assertTrue(e.getMessage().contains("Incompatible schema: exists schema type STRING, new schema type INT8")); + } + } } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java index d5edd96fab563..aa8b54d1a77be 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/protocol/schema/IsCompatibilityResponse.java @@ -33,5 +33,4 @@ public class IsCompatibilityResponse { boolean isCompatibility; String schemaCompatibilityStrategy; - } diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java index ccae1b1176527..18ccddd40d6e9 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java @@ -2355,6 +2355,11 @@ void schemas() throws Exception { PostSchemaPayload input = new ObjectMapper().readValue(new File(schemaFile), PostSchemaPayload.class); verify(schemas).createSchema("persistent://tn1/ns1/tp1", input); + cmdSchemas = new CmdSchemas(() -> admin); + cmdSchemas.run(split("compatibility -f " + schemaFile + " persistent://tn1/ns1/tp1")); + input = new ObjectMapper().readValue(new File(schemaFile), PostSchemaPayload.class); + verify(schemas).testCompatibility("persistent://tn1/ns1/tp1", input); + cmdSchemas = new CmdSchemas(() -> admin); String jarFile = PulsarAdminToolTest.class.getClassLoader() .getResource("dummyexamples.jar").getFile(); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java index 5383e39807b05..44ac143e3507e 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSchemas.java @@ -42,6 +42,7 @@ public CmdSchemas(Supplier admin) { jcommander.addCommand("delete", new DeleteSchema()); jcommander.addCommand("upload", new UploadSchema()); jcommander.addCommand("extract", new ExtractSchema()); + jcommander.addCommand("compatibility", new TestCompatibility()); } @Parameters(commandDescription = "Get the schema for a topic") @@ -164,4 +165,20 @@ void run() throws Exception { } } + @Parameters(commandDescription = "Test schema compatibility") + private class TestCompatibility extends CliCommand { + @Parameter(description = "persistent://tenant/namespace/topic", required = true) + private java.util.List params; + + @Parameter(names = { "-f", "--filename" }, description = "filename", required = true) + private String schemaFileName; + + @Override + void run() throws Exception { + String topic = validateTopicName(params); + PostSchemaPayload input = MAPPER.readValue(new File(schemaFileName), PostSchemaPayload.class); + getAdmin().schemas().testCompatibility(topic, input); + } + } + } From 3a5ffb82eb7be4197db08d25de143b2dc9486269 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 21 Apr 2023 16:06:22 +0800 Subject: [PATCH 054/494] [improve][broker] Move bitmap from lastMutableBucket to ImmutableBucket (#20156) (cherry picked from commit e5a833a2dcb7ce13ada4ca94714cc045a02de276) --- .../bucket/BucketDelayedDeliveryTracker.java | 3 +- .../broker/delayed/bucket/MutableBucket.java | 41 +++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java index c90064c9137bb..67a7de1f01339 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BucketDelayedDeliveryTracker.java @@ -168,7 +168,7 @@ private synchronized long recoverBucketSnapshot() throws RuntimeException { } try { - FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 2, TimeUnit.SECONDS); + FutureUtil.waitForAll(futures.values()).get(AsyncOperationTimeoutSeconds * 5, TimeUnit.SECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("[{}] Failed to recover delayed message index bucket snapshot.", dispatcher.getName(), e); if (e instanceof InterruptedException) { @@ -343,6 +343,7 @@ public synchronized boolean addMessage(long ledgerId, long entryId, long deliver // If (ledgerId < startLedgerId || existBucket) means that message index belong to previous bucket range, // enter sharedBucketPriorityQueue directly sharedBucketPriorityQueue.add(deliverAt, ledgerId, entryId); + lastMutableBucket.putIndexBit(ledgerId, entryId); } else { checkArgument(ledgerId >= lastMutableBucket.endLedgerId); lastMutableBucket.addMessage(ledgerId, entryId, deliverAt); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java index b7e9e68f1bdc7..1173a401a8903 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/MutableBucket.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.delayed.bucket; import static com.google.common.base.Preconditions.checkArgument; -import com.google.protobuf.ByteString; +import com.google.protobuf.UnsafeByteOperations; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -74,6 +74,8 @@ Pair createImmutableBucketAndAsyncPersistent( List bucketSnapshotSegments = new ArrayList<>(); List segmentMetadataList = new ArrayList<>(); + Map immutableBucketBitMap = new HashMap<>(); + Map bitMap = new HashMap<>(); SnapshotSegment snapshotSegment = new SnapshotSegment(); SnapshotSegmentMetadata.Builder segmentMetadataBuilder = SnapshotSegmentMetadata.newBuilder(); @@ -82,18 +84,20 @@ Pair createImmutableBucketAndAsyncPersistent( long currentTimestampUpperLimit = 0; long currentFirstTimestamp = 0L; while (!delayedIndexQueue.isEmpty()) { - DelayedIndex delayedIndex = snapshotSegment.addIndexe(); - delayedIndexQueue.popToObject(delayedIndex); - - long timestamp = delayedIndex.getTimestamp(); + final long timestamp = delayedIndexQueue.peekTimestamp(); if (currentTimestampUpperLimit == 0) { currentFirstTimestamp = timestamp; firstScheduleTimestamps.add(currentFirstTimestamp); currentTimestampUpperLimit = timestamp + timeStepPerBucketSnapshotSegment - 1; } - long ledgerId = delayedIndex.getLedgerId(); - long entryId = delayedIndex.getEntryId(); + DelayedIndex delayedIndex = snapshotSegment.addIndexe(); + delayedIndexQueue.popToObject(delayedIndex); + + final long ledgerId = delayedIndex.getLedgerId(); + final long entryId = delayedIndex.getEntryId(); + + removeIndexBit(ledgerId, entryId); checkArgument(ledgerId >= startLedgerId && ledgerId <= endLedgerId); @@ -102,10 +106,10 @@ Pair createImmutableBucketAndAsyncPersistent( sharedQueue.add(timestamp, ledgerId, entryId); } - numMessages++; - bitMap.computeIfAbsent(ledgerId, k -> new RoaringBitmap()).add(entryId, entryId + 1); + numMessages++; + if (delayedIndexQueue.isEmpty() || delayedIndexQueue.peekTimestamp() > currentTimestampUpperLimit || (maxIndexesPerBucketSnapshotSegment != -1 && snapshotSegment.getIndexesCount() >= maxIndexesPerBucketSnapshotSegment)) { @@ -119,9 +123,17 @@ Pair createImmutableBucketAndAsyncPersistent( final var lId = entry.getKey(); final var bm = entry.getValue(); bm.runOptimize(); - final var array = new byte[bm.serializedSizeInBytes()]; - bm.serialize(ByteBuffer.wrap(array)); - segmentMetadataBuilder.putDelayedIndexBitMap(lId, ByteString.copyFrom(array)); + ByteBuffer byteBuffer = ByteBuffer.allocate(bm.serializedSizeInBytes()); + bm.serialize(byteBuffer); + byteBuffer.flip(); + segmentMetadataBuilder.putDelayedIndexBitMap(lId, UnsafeByteOperations.unsafeWrap(byteBuffer)); + immutableBucketBitMap.compute(lId, (__, bm0) -> { + if (bm0 == null) { + return bm; + } + bm0.or(bm); + return bm0; + }); iterator.remove(); } @@ -133,6 +145,10 @@ Pair createImmutableBucketAndAsyncPersistent( } } + // optimize bm + immutableBucketBitMap.values().forEach(RoaringBitmap::runOptimize); + this.delayedIndexBitMap.values().forEach(RoaringBitmap::runOptimize); + SnapshotMetadata bucketSnapshotMetadata = SnapshotMetadata.newBuilder() .addAllMetadataList(segmentMetadataList) .build(); @@ -145,6 +161,7 @@ Pair createImmutableBucketAndAsyncPersistent( bucket.setNumberBucketDelayedMessages(numMessages); bucket.setLastSegmentEntryId(lastSegmentEntryId); bucket.setFirstScheduleTimestamps(firstScheduleTimestamps); + bucket.setDelayedIndexBitMap(immutableBucketBitMap); // Skip first segment, because it has already been loaded List snapshotSegments = bucketSnapshotSegments.subList(1, bucketSnapshotSegments.size()); From 54382a85a188e700cd6abf1374df571d9eb4b9af Mon Sep 17 00:00:00 2001 From: StevenLuMT Date: Sat, 22 Apr 2023 21:22:26 +0800 Subject: [PATCH 055/494] [fix][test] fix bug for testcase(PersistentTopicsTest#testExamineMessage) (#20154) Co-authored-by: lushiji (cherry picked from commit c58da14968f27cf2a4813db01f387b61a5a03636) --- .../org/apache/pulsar/broker/admin/PersistentTopicsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 6949fe931ca5a..f80d9863a26a6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -1144,6 +1144,7 @@ public void testExamineMessage() throws Exception { // Check examine message not allowed on partitioned topic. try { admin.topics().examineMessage(topicName, "earliest", 1); + Assert.fail("fail to check examine message not allowed on partitioned topic"); } catch (PulsarAdminException e) { Assert.assertEquals(e.getMessage(), "Examine messages on a partitioned topic is not allowed, please try examine message on specific " From 214a6bd3f55048ebd39efdc999a801b52fa8a0a1 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 23 Apr 2023 09:41:28 +0800 Subject: [PATCH 056/494] [fix] [broker] Fast fix infinite HTTP call getSubscriptions caused by wrong topicName (#20131) (cherry picked from commit 0c50866fbc18525f82a04c3a918628b8b50a4de8) --- .../admin/impl/PersistentTopicsBase.java | 32 +++- ...rInfiniteHttpCallGetSubscriptionsTest.java | 143 ++++++++++++++++++ 2 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 7347d6dbf20a2..6b163ae04469b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionCoordinatorAssign; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; +import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectReader; import com.github.zafarkhaja.semver.Version; @@ -1171,6 +1172,21 @@ protected CompletableFuture internalDeleteTopicAsync(boolean authoritative .thenCompose(__ -> pulsar().getBrokerService().deleteTopic(topicName.toString(), force)); } + /** + * There has a known bug will make Pulsar misidentifies "tp-partition-0-DLQ-partition-0" as "tp-partition-0-DLQ". + * You can see the details from PR https://github.com/apache/pulsar/pull/19841. + * This method is a quick fix and will be removed in master branch after #19841 and PIP 263 are done. + */ + private boolean isUnexpectedTopicName(PartitionedTopicMetadata topicMetadata) { + if (!topicName.toString().contains(PARTITIONED_TOPIC_SUFFIX)){ + return false; + } + if (topicMetadata.partitions <= 0){ + return false; + } + return topicName.getPartition(0).toString().equals(topicName.toString()); + } + protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean authoritative) { CompletableFuture future; if (topicName.isGlobal()) { @@ -1188,7 +1204,7 @@ protected void internalGetSubscriptions(AsyncResponse asyncResponse, boolean aut } else { getPartitionedTopicMetadataAsync(topicName, authoritative, false) .thenAccept(partitionMetadata -> { - if (partitionMetadata.partitions > 0) { + if (partitionMetadata.partitions > 0 && !isUnexpectedTopicName(partitionMetadata)) { try { final Set subscriptions = Collections.newSetFromMap( @@ -3716,7 +3732,7 @@ protected CompletableFuture preValidation(boolean authoritative) { .thenCompose(metadata -> { if (metadata.partitions > 0) { return validateTopicOwnershipAsync(TopicName.get(topicName.toString() - + TopicName.PARTITIONED_TOPIC_SUFFIX + 0), authoritative); + + PARTITIONED_TOPIC_SUFFIX + 0), authoritative); } else { return validateTopicOwnershipAsync(topicName, authoritative); } @@ -4543,7 +4559,7 @@ protected CompletableFuture internalValidateClientVersionAsync() { private CompletableFuture validatePartitionTopicUpdateAsync(String topicName, int numberOfPartition) { return internalGetListAsync().thenCompose(existingTopicList -> { TopicName partitionTopicName = TopicName.get(domain(), namespaceName, topicName); - String prefix = partitionTopicName.getPartitionedTopicName() + TopicName.PARTITIONED_TOPIC_SUFFIX; + String prefix = partitionTopicName.getPartitionedTopicName() + PARTITIONED_TOPIC_SUFFIX; return getPartitionedTopicMetadataAsync(partitionTopicName, false, false) .thenAccept(metadata -> { int oldPartition = metadata.partitions; @@ -4551,8 +4567,8 @@ private CompletableFuture validatePartitionTopicUpdateAsync(String topicNa if (existingTopicName.startsWith(prefix)) { try { long suffix = Long.parseLong(existingTopicName.substring( - existingTopicName.indexOf(TopicName.PARTITIONED_TOPIC_SUFFIX) - + TopicName.PARTITIONED_TOPIC_SUFFIX.length())); + existingTopicName.indexOf(PARTITIONED_TOPIC_SUFFIX) + + PARTITIONED_TOPIC_SUFFIX.length())); // Skip partition of partitioned topic by making sure // the numeric suffix greater than old partition number. if (suffix >= oldPartition && suffix <= (long) numberOfPartition) { @@ -4593,12 +4609,12 @@ private CompletableFuture validatePartitionTopicUpdateAsync(String topicNa */ private CompletableFuture validateNonPartitionTopicNameAsync(String topicName) { CompletableFuture ret = CompletableFuture.completedFuture(null); - if (topicName.contains(TopicName.PARTITIONED_TOPIC_SUFFIX)) { + if (topicName.contains(PARTITIONED_TOPIC_SUFFIX)) { try { // First check if what's after suffix "-partition-" is number or not, if not number then can create. - int partitionIndex = topicName.indexOf(TopicName.PARTITIONED_TOPIC_SUFFIX); + int partitionIndex = topicName.indexOf(PARTITIONED_TOPIC_SUFFIX); long suffix = Long.parseLong(topicName.substring(partitionIndex - + TopicName.PARTITIONED_TOPIC_SUFFIX.length())); + + PARTITIONED_TOPIC_SUFFIX.length())); TopicName partitionTopicName = TopicName.get(domain(), namespaceName, topicName.substring(0, partitionIndex)); ret = getPartitionedTopicMetadataAsync(partitionTopicName, false, false) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java new file mode 100644 index 0000000000000..2efc4f4e7803f --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicType; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class TopicNameForInfiniteHttpCallGetSubscriptionsTest extends ProducerConsumerBase { + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED); + conf.setDefaultNumPartitions(1); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testInfiniteHttpCallGetSubscriptions() throws Exception { + final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); + final String partitionedTopicName = "persistent://my-property/my-ns/tp1_" + randomStr; + final String topic_p0 = partitionedTopicName + TopicName.PARTITIONED_TOPIC_SUFFIX + "0"; + final String subscriptionName = "sub1"; + final String topicDLQ = topic_p0 + "-" + subscriptionName + "-DLQ"; + + admin.topics().createPartitionedTopic(partitionedTopicName, 2); + + // Do test. + ProducerAndConsumerEntry pcEntry = triggerDLQCreated(topic_p0, topicDLQ, subscriptionName); + admin.topics().getSubscriptions(topicDLQ); + + // cleanup. + pcEntry.consumer.close(); + pcEntry.producer.close(); + admin.topics().deletePartitionedTopic(partitionedTopicName); + } + + @Test + public void testInfiniteHttpCallGetSubscriptions2() throws Exception { + final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); + final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0-abc"; + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + + // Do test. + admin.topics().getSubscriptions(topicName); + + // cleanup. + producer.close(); + } + + @Test + public void testInfiniteHttpCallGetSubscriptions3() throws Exception { + final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); + final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0"; + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + + // Do test. + admin.topics().getSubscriptions(topicName); + + // cleanup. + producer.close(); + } + + @AllArgsConstructor + private static class ProducerAndConsumerEntry { + private Producer producer; + private Consumer consumer; + } + + private ProducerAndConsumerEntry triggerDLQCreated(String topicName, String DLQName, String subscriptionName) throws Exception { + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .enableRetry(true) + .deadLetterPolicy(DeadLetterPolicy.builder().deadLetterTopic(DLQName).maxRedeliverCount(2).build()) + .receiverQueueSize(100) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .create(); + // send messages. + for (int i = 0; i < 5; i++) { + producer.newMessage() + .value("value-" + i) + .sendAsync(); + } + producer.flush(); + // trigger the DLQ created. + for (int i = 0; i < 20; i++) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + if (msg != null) { + consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS); + } else { + break; + } + } + + return new ProducerAndConsumerEntry(producer, consumer); + } +} \ No newline at end of file From cde687a8e853b6944e5f8942bcbb9fb29f80e632 Mon Sep 17 00:00:00 2001 From: WangJialing <65590138+wangjialing218@users.noreply.github.com> Date: Sun, 23 Apr 2023 10:07:02 +0800 Subject: [PATCH 057/494] [fix][broker] Fix getPartitionedStats miss subscription's messageAckRate (#19870) (cherry picked from commit d3158bfebf9565ea9422a9116749628648e9f90d) --- .../common/policies/data/stats/SubscriptionStatsImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index 25fa666523f3c..7ae439829b319 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -160,6 +160,8 @@ public void reset() { bytesOutCounter = 0; msgOutCounter = 0; msgRateRedeliver = 0; + messageAckRate = 0; + chunkedMessageRate = 0; msgBacklog = 0; backlogSize = 0; msgBacklogNoDelayed = 0; @@ -190,6 +192,8 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) { this.bytesOutCounter += stats.bytesOutCounter; this.msgOutCounter += stats.msgOutCounter; this.msgRateRedeliver += stats.msgRateRedeliver; + this.messageAckRate += stats.messageAckRate; + this.chunkedMessageRate += stats.chunkedMessageRate; this.msgBacklog += stats.msgBacklog; this.backlogSize += stats.backlogSize; this.msgBacklogNoDelayed += stats.msgBacklogNoDelayed; From 43e81029c98fa72070200684423a6baabada7849 Mon Sep 17 00:00:00 2001 From: tiny-rain Date: Sun, 23 Apr 2023 13:45:03 +0800 Subject: [PATCH 058/494] [fix][meta] deadlock of zkSessionWatcher when zkConnection loss (#20122) (cherry picked from commit d3d36bdbd22a1b1fa6cee41a91acd9ec8a4c9341) --- .../org/apache/pulsar/metadata/impl/ZKSessionWatcher.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java index 1ce01f57d4fbe..a840721023080 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKSessionWatcher.java @@ -81,7 +81,11 @@ public void close() throws Exception { } // task that runs every TICK_TIME to check zk connection - private synchronized void checkConnectionStatus() { + // NOT ThreadSafe: + // If zk client can't ensure the order, it may lead to problems. + // Currently,we only use it in single thread, it will be fine. but we shouldn't leave any potential problems + // in the future. + private void checkConnectionStatus() { try { CompletableFuture future = new CompletableFuture<>(); zk.exists("/", false, (StatCallback) (rc, path, ctx, stat) -> { @@ -126,7 +130,7 @@ synchronized void setSessionInvalid() { currentStatus = SessionEvent.SessionLost; } - private void checkState(Watcher.Event.KeeperState zkClientState) { + private synchronized void checkState(Watcher.Event.KeeperState zkClientState) { switch (zkClientState) { case Expired: if (currentStatus != SessionEvent.SessionLost) { From 66f04fafa46a9ca96861785ae8a086b6c2886a6b Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 24 Apr 2023 00:26:23 +0800 Subject: [PATCH 059/494] [fix][broker] Remove useless field in the DelayedMessageIndexBucketSegment.proto (#20166) (cherry picked from commit ab8a8c9ef58c0d3b7c0838df81f9762637c24f90) --- .../src/main/proto/DelayedMessageIndexBucketSegment.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto index 633d6a8f1615c..a6ed30cfe8cd4 100644 --- a/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto +++ b/pulsar-broker/src/main/proto/DelayedMessageIndexBucketSegment.proto @@ -31,5 +31,4 @@ message DelayedIndex { message SnapshotSegment { repeated DelayedIndex indexes = 1; - map delayed_index_bit_map = 2; } From 6ca4029af151903d78c9dd46f099761e54c726b6 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 24 Apr 2023 10:34:20 +0800 Subject: [PATCH 060/494] [fix][broker] Release EntryBuffer after parse proto object (#20170) (cherry picked from commit 35e9897742b7db4bd29349940075a819b2ad6999) --- .../BookkeeperBucketSnapshotStorage.java | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java index 040bbbc586f49..e99f39b382f56 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/delayed/bucket/BookkeeperBucketSnapshotStorage.java @@ -20,6 +20,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -38,6 +39,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.delayed.proto.SnapshotMetadata; import org.apache.pulsar.broker.delayed.proto.SnapshotSegment; +import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.util.FutureUtil; @Slf4j @@ -60,8 +62,9 @@ public BookkeeperBucketSnapshotStorage(PulsarService pulsar) { public CompletableFuture createBucketSnapshot(SnapshotMetadata snapshotMetadata, List bucketSnapshotSegments, String bucketKey, String topicName, String cursorName) { + ByteBuf metadataByteBuf = Unpooled.wrappedBuffer(snapshotMetadata.toByteArray()); return createLedger(bucketKey, topicName, cursorName) - .thenCompose(ledgerHandle -> addEntry(ledgerHandle, snapshotMetadata.toByteArray()) + .thenCompose(ledgerHandle -> addEntry(ledgerHandle, metadataByteBuf) .thenCompose(__ -> addSnapshotSegments(ledgerHandle, bucketSnapshotSegments)) .thenCompose(__ -> closeLedger(ledgerHandle)) .thenApply(__ -> ledgerHandle.getId())); @@ -117,19 +120,32 @@ public void close() throws Exception { private CompletableFuture addSnapshotSegments(LedgerHandle ledgerHandle, List bucketSnapshotSegments) { List> addFutures = new ArrayList<>(); + ByteBuf byteBuf; for (SnapshotSegment bucketSnapshotSegment : bucketSnapshotSegments) { - addFutures.add(addEntry(ledgerHandle, bucketSnapshotSegment.toByteArray())); + byteBuf = PulsarByteBufAllocator.DEFAULT.directBuffer(bucketSnapshotSegment.getSerializedSize()); + try { + bucketSnapshotSegment.writeTo(byteBuf); + } catch (Exception e){ + byteBuf.release(); + throw e; + } + addFutures.add(addEntry(ledgerHandle, byteBuf)); } return FutureUtil.waitForAll(addFutures); } private SnapshotMetadata parseSnapshotMetadataEntry(LedgerEntry ledgerEntry) { + ByteBuf entryBuffer = null; try { - ByteBuf entryBuffer = ledgerEntry.getEntryBuffer(); + entryBuffer = ledgerEntry.getEntryBuffer(); return SnapshotMetadata.parseFrom(entryBuffer.nioBuffer()); } catch (InvalidProtocolBufferException e) { throw new BucketSnapshotSerializationException(e); + } finally { + if (entryBuffer != null) { + entryBuffer.release(); + } } } @@ -139,7 +155,11 @@ private List parseSnapshotSegmentEntries(Enumeration closeLedger(LedgerHandle ledgerHandle) { return future; } - private CompletableFuture addEntry(LedgerHandle ledgerHandle, byte[] data) { + private CompletableFuture addEntry(LedgerHandle ledgerHandle, ByteBuf data) { final CompletableFuture future = new CompletableFuture<>(); ledgerHandle.asyncAddEntry(data, (rc, handle, entryId, ctx) -> { From 29eee9495cea82cc3547d238d73611c246adae35 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 26 Apr 2023 22:16:39 +0800 Subject: [PATCH 061/494] [fix][test] Fix flaky test `ConcurrentBitmapSortedLongPairSetTest` (#20165) (cherry picked from commit 0ec576b25fdd7e4890799184ac03f66deb4735f6) --- .../utils/ConcurrentBitmapSortedLongPairSetTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java index 47379c3b6f17c..5f8f13288cfe8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/utils/ConcurrentBitmapSortedLongPairSetTest.java @@ -26,7 +26,6 @@ import org.testng.annotations.Test; import java.util.ArrayList; import java.util.List; -import java.util.Random; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -188,15 +187,12 @@ public void concurrentInsertions() throws Throwable { List> futures = new ArrayList<>(); for (int i = 0; i < nThreads; i++) { final int threadIdx = i; - futures.add(executor.submit(() -> { - Random random = new Random(); + int start = N * (threadIdx + 1); for (int j = 0; j < N; j++) { - int key = random.nextInt(); + int key = start + j; // Ensure keys are unique - key -= key % (threadIdx + 1); - key = Math.abs(key); set.add(key, key); } })); From 8da40f0f033cfad7d7d8a2b005ced69e549f7e86 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Fri, 28 Apr 2023 16:46:51 +0800 Subject: [PATCH 062/494] [fix][admin] Fix examine messages if total message is zero (#20152) (cherry picked from commit 8b7aa6312b2af49d0c6edde85c6b0108007a0a96) --- .../pulsar/broker/admin/impl/PersistentTopicsBase.java | 4 ++++ .../apache/pulsar/broker/admin/PersistentTopicsTest.java | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 6b163ae04469b..fcade8270cb12 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -3061,6 +3061,10 @@ protected CompletableFuture internalExamineMessageAsync(String initial try { PersistentTopic persistentTopic = (PersistentTopic) topic; long totalMessage = persistentTopic.getNumberOfEntries(); + if (totalMessage <= 0) { + throw new RestException(Status.PRECONDITION_FAILED, + "Could not examine messages due to the total message is zero"); + } PositionImpl startPosition = persistentTopic.getFirstPosition(); long messageToSkip = initialPositionLocal.equals("earliest") ? messagePositionLocal : diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index f80d9863a26a6..284e50c830286 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -1151,6 +1151,14 @@ public void testExamineMessage() throws Exception { + "topic partition"); } + try { + admin.topics().examineMessage(topicName + "-partition-0", "earliest", 1); + Assert.fail(); + } catch (PulsarAdminException e) { + Assert.assertEquals(e.getMessage(), + "Could not examine messages due to the total message is zero"); + } + producer.send("message1"); Assert.assertEquals( new String(admin.topics().examineMessage(topicName + "-partition-0", "earliest", 1).getData()), From 89dafa5e03129b4f35db31232fd816acd454cf11 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 29 Apr 2023 01:54:05 +0800 Subject: [PATCH 063/494] [improve] [broker] Skip split boundle if only one broker (#20190) Co-authored-by: Zixuan Liu (cherry picked from commit d135c4a115038dc61f8fe2d230cb1f0c02239f92) --- .../impl/ModularLoadManagerImpl.java | 2 +- .../broker/stats/PrometheusMetricsTest.java | 9 +++ .../client/api/BrokerServiceLookupTest.java | 58 +++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index c6f406a7c8a5c..30a2ef5cdf250 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -744,7 +744,7 @@ public boolean shouldAntiAffinityNamespaceUnload(String namespace, String bundle public void checkNamespaceBundleSplit() { if (!conf.isLoadBalancerAutoBundleSplitEnabled() || pulsar.getLeaderElectionService() == null - || !pulsar.getLeaderElectionService().isLeader()) { + || !pulsar.getLeaderElectionService().isLeader() || knownBrokers.size() <= 1) { return; } final boolean unloadSplitBundles = pulsar.getConfiguration().isLoadBalancerAutoUnloadSplitBundlesEnabled(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index f5ae8459f1803..bf141e10aa1b9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -80,6 +81,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.compaction.Compactor; +import org.apache.zookeeper.CreateMode; import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; @@ -772,6 +774,10 @@ public void testBundlesMetrics() throws Exception { c1.acknowledge(c1.receive()); } + // Mock another broker to make split task work. + String mockedBroker = "/loadbalance/brokers/127.0.0.1:0"; + mockZooKeeper.create(mockedBroker, new byte[]{0}, Collections.emptyList(), CreateMode.EPHEMERAL); + pulsar.getBrokerService().updateRates(); Awaitility.await().untilAsserted(() -> assertTrue(pulsar.getBrokerService().getBundleStats().size() > 0)); ModularLoadManagerWrapper loadManager = (ModularLoadManagerWrapper)pulsar.getLoadManager().get(); @@ -796,6 +802,9 @@ public void testBundlesMetrics() throws Exception { assertTrue(metrics.containsKey("pulsar_lb_bandwidth_out_usage")); assertTrue(metrics.containsKey("pulsar_lb_bundles_split_total")); + + // cleanup. + mockZooKeeper.delete(mockedBroker, 0); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index 2af9f450dd3b1..8597e0a87997a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -67,6 +67,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; @@ -80,6 +81,7 @@ import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; @@ -781,6 +783,62 @@ public void testModularLoadManagerSplitBundle() throws Exception { } } + @Test(timeOut = 20000) + public void testSkipSplitBundleIfOnlyOneBroker() throws Exception { + + log.info("-- Starting {} test --", methodName); + final String loadBalancerName = conf.getLoadManagerClassName(); + final int defaultNumberOfNamespaceBundles = conf.getDefaultNumberOfNamespaceBundles(); + final int loadBalancerNamespaceBundleMaxTopics = conf.getLoadBalancerNamespaceBundleMaxTopics(); + + final String namespace = "my-property/my-ns"; + final String topicName1 = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_"); + final String topicName2 = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_"); + try { + // configure broker with ModularLoadManager. + stopBroker(); + conf.setDefaultNumberOfNamespaceBundles(1); + conf.setLoadBalancerNamespaceBundleMaxTopics(1); + conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + startBroker(); + final ModularLoadManagerWrapper modularLoadManagerWrapper = + (ModularLoadManagerWrapper) pulsar.getLoadManager().get(); + final ModularLoadManagerImpl modularLoadManager = + (ModularLoadManagerImpl) modularLoadManagerWrapper.getLoadManager(); + + // Create one topic and trigger tasks, then verify there is only one bundle now. + Consumer consumer1 = pulsarClient.newConsumer().topic(topicName1) + .subscriptionName("my-subscriber-name").subscribe(); + List bounldes1 = pulsar.getNamespaceService().getNamespaceBundleFactory() + .getBundles(NamespaceName.get(namespace)).getBundles(); + pulsar.getBrokerService().updateRates(); + pulsar.getLoadManager().get().writeLoadReportOnZookeeper(); + pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper(); + modularLoadManager.updateAll(); + assertEquals(bounldes1.size(), 1); + + // Create the second topic and trigger tasks, then verify the split task will be skipped. + Consumer consumer2 = pulsarClient.newConsumer().topic(topicName2) + .subscriptionName("my-subscriber-name").subscribe(); + pulsar.getBrokerService().updateRates(); + pulsar.getLoadManager().get().writeLoadReportOnZookeeper(); + pulsar.getLoadManager().get().writeResourceQuotasToZooKeeper(); + modularLoadManager.updateAll(); + List bounldes2 = pulsar.getNamespaceService().getNamespaceBundleFactory() + .getBundles(NamespaceName.get(namespace)).getBundles(); + assertEquals(bounldes2.size(), 1); + + consumer1.close(); + consumer2.close(); + admin.topics().delete(topicName1, false); + admin.topics().delete(topicName2, false); + } finally { + conf.setDefaultNumberOfNamespaceBundles(defaultNumberOfNamespaceBundles); + conf.setLoadBalancerNamespaceBundleMaxTopics(loadBalancerNamespaceBundleMaxTopics); + conf.setLoadManagerClassName(loadBalancerName); + } + } + @Test(timeOut = 10000) public void testPartitionedMetadataWithDeprecatedVersion() throws Exception { From e184605f15fae02e088a5f9d685a3ce24006b541 Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Sun, 30 Apr 2023 12:58:53 +0300 Subject: [PATCH 064/494] [fix][docs] Remove old template inlined (#20208) (cherry picked from commit e8d63952a40b79499eb3fd524f80db6bc986ed34) --- wiki/proposals/PIP.md | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/wiki/proposals/PIP.md b/wiki/proposals/PIP.md index 159c9199895d1..fc68905726eb8 100644 --- a/wiki/proposals/PIP.md +++ b/wiki/proposals/PIP.md @@ -116,38 +116,5 @@ Some examples: ## Template for a PIP design doc -``` -## Motivation +Read [the template file](/.github/ISSUE_TEMPLATE/pip.md). -Explain why this change is needed, what benefits it would bring to Apache Pulsar -and what problem it's trying to solve. - -## Goal - -Define the scope of this proposal. Given the motivation stated above, what are -the problems that this proposal is addressing and what other items will be -considering out of scope, perhaps to be left to a different PIP. - -## API Changes - -Illustrate all the proposed changes to the API or wire protocol, with examples -of all the newly added classes/methods, including Javadoc. - -## Implementation - -This should be a detailed description of all the changes that are -expected to be made. It should be detailed enough that any developer that is -familiar with Pulsar internals would be able to understand all the parts of the -code changes for this proposal. - -This should also serve as documentation for any person that is trying to -understand or debug the behavior of a certain feature. - - -## Reject Alternatives - -If there are alternatives that were already considered by the authors or, -after the discussion, by the community, and were rejected, please list them -here along with the reason why they were rejected. - -``` From 21c2eddeb8a912b5ad88f7c4071d6e3f71627c22 Mon Sep 17 00:00:00 2001 From: ran Date: Fri, 5 May 2023 08:48:59 +0800 Subject: [PATCH 065/494] [fix][broker] Fix the reason label of authentication metrics (#20030) (cherry picked from commit 2b515ffb389c4b4fe3cb5a9c5f3d7eb0a4c9ef99) --- .../AuthenticationProviderAthenz.java | 20 ++++++++++-- .../oidc/AuthenticationProviderOpenID.java | 2 +- .../AuthenticationProvider.java | 5 +++ .../AuthenticationProviderBasic.java | 21 +++++++++++-- .../AuthenticationProviderList.java | 20 +++++++----- .../AuthenticationProviderTls.java | 12 +++++-- .../AuthenticationProviderToken.java | 31 ++++++++++++------- .../metrics/AuthenticationMetrics.java | 16 ++++++++++ .../broker/stats/PrometheusMetricsTest.java | 6 ++-- 9 files changed, 103 insertions(+), 30 deletions(-) diff --git a/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java b/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java index 2e062b87a8325..652a922b9a5ad 100644 --- a/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java +++ b/pulsar-broker-auth-athenz/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenz.java @@ -43,6 +43,15 @@ public class AuthenticationProviderAthenz implements AuthenticationProvider { private List domainNameList = null; private int allowedOffset = 30; + public enum ErrorCode { + UNKNOWN, + NO_CLIENT, + NO_TOKEN, + NO_PUBLIC_KEY, + DOMAIN_MISMATCH, + INVALID_TOKEN, + } + @Override public void initialize(ServiceConfiguration config) throws IOException { String domainNames; @@ -81,11 +90,13 @@ public String getAuthMethodName() { public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { SocketAddress clientAddress; String roleToken; + ErrorCode errorCode = ErrorCode.UNKNOWN; try { if (authData.hasDataFromPeer()) { clientAddress = authData.getPeerAddress(); } else { + errorCode = ErrorCode.NO_CLIENT; throw new AuthenticationException("Authentication data source does not have a client address"); } @@ -94,13 +105,16 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat } else if (authData.hasDataFromHttp()) { roleToken = authData.getHttpHeader(AuthZpeClient.ZPE_TOKEN_HDR); } else { + errorCode = ErrorCode.NO_TOKEN; throw new AuthenticationException("Authentication data source does not have a role token"); } if (roleToken == null) { + errorCode = ErrorCode.NO_TOKEN; throw new AuthenticationException("Athenz token is null, can't authenticate"); } if (roleToken.isEmpty()) { + errorCode = ErrorCode.NO_TOKEN; throw new AuthenticationException("Athenz RoleToken is empty, Server is Using Athenz Authentication"); } if (log.isDebugEnabled()) { @@ -110,6 +124,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat RoleToken token = new RoleToken(roleToken); if (!domainNameList.contains(token.getDomain())) { + errorCode = ErrorCode.DOMAIN_MISMATCH; throw new AuthenticationException( String.format("Athenz RoleToken Domain mismatch, Expected: %s, Found: %s", domainNameList.toString(), token.getDomain())); @@ -120,6 +135,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat PublicKey ztsPublicKey = AuthZpeClient.getZtsPublicKey(token.getKeyId()); if (ztsPublicKey == null) { + errorCode = ErrorCode.NO_PUBLIC_KEY; throw new AuthenticationException("Unable to retrieve ZTS Public Key"); } @@ -128,13 +144,13 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); return token.getPrincipal(); } else { + errorCode = ErrorCode.INVALID_TOKEN; throw new AuthenticationException( String.format("Athenz Role Token Not Authenticated from Client: %s", clientAddress)); } } } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), - exception.getMessage()); + incrementFailureMetric(errorCode); throw exception; } } diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java index ae4774b6f6b6b..00ec09bd1817f 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java @@ -447,7 +447,7 @@ DecodedJWT verifyJWT(PublicKey publicKey, } static void incrementFailureMetric(AuthenticationExceptionCode code) { - AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, "token", code.toString()); + AuthenticationMetrics.authenticateFailure(SIMPLE_NAME, AUTH_METHOD_NAME, code); } /** diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java index 109259537a494..7862a35b5e871 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProvider.java @@ -30,6 +30,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.metrics.AuthenticationMetrics; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.util.FutureUtil; @@ -143,6 +144,10 @@ default CompletableFuture authenticateHttpRequestAsync(HttpServletReque } } + default void incrementFailureMetric(Enum errorCode) { + AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), errorCode); + } + /** * Set response, according to passed in request. * and return whether we should do following chain.doFilter or not. diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java index 3c4759fec7da9..ca5150c9bdb60 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderBasic.java @@ -46,6 +46,14 @@ public class AuthenticationProviderBasic implements AuthenticationProvider { private static final String CONF_PULSAR_PROPERTY_KEY = "basicAuthConf"; private Map users; + private enum ErrorCode { + UNKNOWN, + EMPTY_AUTH_DATA, + INVALID_HEADER, + INVALID_AUTH_DATA, + INVALID_TOKEN, + } + @Override public void close() throws IOException { // noop @@ -104,9 +112,10 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat String userId = authParams.getUserId(); String password = authParams.getPassword(); String msg = "Unknown user or invalid password"; - + ErrorCode errorCode = ErrorCode.UNKNOWN; try { if (users.get(userId) == null) { + errorCode = ErrorCode.INVALID_AUTH_DATA; throw new AuthenticationException(msg); } @@ -117,15 +126,16 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat List splitEncryptedPassword = Arrays.asList(encryptedPassword.split("\\$")); if (splitEncryptedPassword.size() != 4 || !encryptedPassword .equals(Md5Crypt.apr1Crypt(password.getBytes(), splitEncryptedPassword.get(2)))) { + errorCode = ErrorCode.INVALID_TOKEN; throw new AuthenticationException(msg); } // For crypt algorithm } else if (!encryptedPassword.equals(Crypt.crypt(password.getBytes(), encryptedPassword.substring(0, 2)))) { + errorCode = ErrorCode.INVALID_TOKEN; throw new AuthenticationException(msg); } } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), - exception.getMessage()); + incrementFailureMetric(errorCode); throw exception; } AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); @@ -144,24 +154,29 @@ public AuthParams(AuthenticationDataSource authData) throws AuthenticationExcept String rawAuthToken = authData.getHttpHeader(HTTP_HEADER_NAME); // parsing and validation if (StringUtils.isBlank(rawAuthToken) || !rawAuthToken.toUpperCase().startsWith("BASIC ")) { + incrementFailureMetric(ErrorCode.INVALID_HEADER); throw new AuthenticationException("Authentication token has to be started with \"Basic \""); } String[] splitRawAuthToken = rawAuthToken.split(" "); if (splitRawAuthToken.length != 2) { + incrementFailureMetric(ErrorCode.INVALID_HEADER); throw new AuthenticationException("Base64 encoded token is not found"); } try { authParams = new String(Base64.getDecoder().decode(splitRawAuthToken[1])); } catch (Exception e) { + incrementFailureMetric(ErrorCode.INVALID_HEADER); throw new AuthenticationException("Base64 decoding is failure: " + e.getMessage()); } } else { + incrementFailureMetric(ErrorCode.EMPTY_AUTH_DATA); throw new AuthenticationException("Authentication data source does not have data"); } String[] parsedAuthParams = authParams.split(":"); if (parsedAuthParams.length != 2) { + incrementFailureMetric(ErrorCode.INVALID_AUTH_DATA); throw new AuthenticationException("Base64 decoded params are invalid"); } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java index 16d6f9859a0b4..663a6253f4460 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderList.java @@ -44,9 +44,15 @@ private interface AuthProcessor { } + private enum ErrorCode { + UNKNOWN, + AUTH_REQUIRED, + } + static T applyAuthProcessor(List processors, AuthProcessor authFunc) throws AuthenticationException { AuthenticationException authenticationException = null; + String errorCode = ErrorCode.UNKNOWN.name(); for (W ap : processors) { try { return authFunc.apply(ap); @@ -56,19 +62,19 @@ static T applyAuthProcessor(List processors, AuthProcessor authF } // Store the exception so we can throw it later instead of a generic one authenticationException = ae; + errorCode = ap.getClass().getSimpleName() + "-INVALID-AUTH"; } } if (null == authenticationException) { AuthenticationMetrics.authenticateFailure( AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", "Authentication required"); + "authentication-provider-list", ErrorCode.AUTH_REQUIRED); throw new AuthenticationException("Authentication required"); } else { - AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", - authenticationException.getMessage() != null - ? authenticationException.getMessage() : "Authentication required"); + AuthenticationMetrics.authenticateFailure( + AuthenticationProviderList.class.getSimpleName(), + "authentication-provider-list", errorCode); throw authenticationException; } @@ -129,7 +135,7 @@ private void authenticateRemainingAuthStates(CompletableFuture authCha previousException = new AuthenticationException("Authentication required"); } AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", "Authentication required"); + "authentication-provider-list", ErrorCode.AUTH_REQUIRED); authChallengeFuture.completeExceptionally(previousException); return; } @@ -235,7 +241,7 @@ private void authenticateRemainingAuthProviders(CompletableFuture roleFu previousException = new AuthenticationException("Authentication required"); } AuthenticationMetrics.authenticateFailure(AuthenticationProviderList.class.getSimpleName(), - "authentication-provider-list", "Authentication required"); + "authentication-provider-list", ErrorCode.AUTH_REQUIRED); roleFuture.completeExceptionally(previousException); return; } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java index 47e5316bce9ce..a4c44121b4b96 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderTls.java @@ -27,6 +27,12 @@ public class AuthenticationProviderTls implements AuthenticationProvider { + private enum ErrorCode { + UNKNOWN, + INVALID_CERTS, + INVALID_CN, // invalid common name + } + @Override public void close() throws IOException { // noop @@ -45,6 +51,7 @@ public String getAuthMethodName() { @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { String commonName = null; + ErrorCode errorCode = ErrorCode.UNKNOWN; try { if (authData.hasDataFromTls()) { /** @@ -72,6 +79,7 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat // CN=Steve Kille,O=Isode Limited,C=GB Certificate[] certs = authData.getTlsCertificates(); if (null == certs) { + errorCode = ErrorCode.INVALID_CERTS; throw new AuthenticationException("Failed to get TLS certificates from client"); } String distinguishedName = ((X509Certificate) certs[0]).getSubjectX500Principal().getName(); @@ -85,12 +93,12 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat } if (commonName == null) { + errorCode = ErrorCode.INVALID_CN; throw new AuthenticationException("Client unable to authenticate with TLS certificate"); } AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), - exception.getMessage()); + incrementFailureMetric(errorCode); throw exception; } return commonName; diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java index 67e1f39a38924..f8992b21ff49f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/AuthenticationProviderToken.java @@ -106,6 +106,12 @@ public class AuthenticationProviderToken implements AuthenticationProvider { private String confTokenAudienceSettingName; private String confTokenAllowedClockSkewSecondsSettingName; + public enum ErrorCode { + INVALID_AUTH_DATA, + INVALID_TOKEN, + INVALID_AUDIENCES, + } + @Override public void close() throws IOException { // noop @@ -158,19 +164,18 @@ public String getAuthMethodName() { @Override public String authenticate(AuthenticationDataSource authData) throws AuthenticationException { + String token; try { // Get Token - String token; token = getToken(authData); - // Parse Token by validating - String role = getPrincipal(authenticateToken(token)); - AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); - return role; } catch (AuthenticationException exception) { - AuthenticationMetrics.authenticateFailure(getClass().getSimpleName(), getAuthMethodName(), - exception.getMessage()); + incrementFailureMetric(ErrorCode.INVALID_AUTH_DATA); throw exception; } + // Parse Token by validating + String role = getPrincipal(authenticateToken(token)); + AuthenticationMetrics.authenticateSuccess(getClass().getSimpleName(), getAuthMethodName()); + return role; } @Override @@ -241,16 +246,19 @@ private Jwt authenticateToken(final String token) throws Authenticati List audiences = (List) object; // audience not contains this broker, throw exception. if (audiences.stream().noneMatch(audienceInToken -> audienceInToken.equals(audience))) { - throw new AuthenticationException("Audiences in token: [" + String.join(", ", audiences) - + "] not contains this broker: " + audience); + incrementFailureMetric(ErrorCode.INVALID_AUDIENCES); + throw new AuthenticationException("Audiences in token: [" + + String.join(", ", audiences) + "] not contains this broker: " + audience); } } else if (object instanceof String) { if (!object.equals(audience)) { - throw new AuthenticationException("Audiences in token: [" + object - + "] not contains this broker: " + audience); + incrementFailureMetric(ErrorCode.INVALID_AUDIENCES); + throw new AuthenticationException( + "Audiences in token: [" + object + "] not contains this broker: " + audience); } } else { // should not reach here. + incrementFailureMetric(ErrorCode.INVALID_AUDIENCES); throw new AuthenticationException("Audiences in token is not in expected format: " + object); } } @@ -264,6 +272,7 @@ private Jwt authenticateToken(final String token) throws Authenticati if (e instanceof ExpiredJwtException) { expiredTokenMetrics.inc(); } + incrementFailureMetric(ErrorCode.INVALID_TOKEN); throw new AuthenticationException("Failed to authentication token: " + e.getMessage()); } } diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java index 0a095235f3cb3..5faaccbe15716 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authentication/metrics/AuthenticationMetrics.java @@ -43,11 +43,27 @@ public static void authenticateSuccess(String providerName, String authMethod) { /** * Log authenticate failure event to the authentication metrics. + * + * This method is deprecated due to the label "reason" is a potential infinite value. + * @deprecated See {@link #authenticateFailure(String, String, Enum)} ()} + * * @param providerName The short class name of the provider * @param authMethod Authentication method name. * @param reason Failure reason. */ + @Deprecated public static void authenticateFailure(String providerName, String authMethod, String reason) { authFailuresMetrics.labels(providerName, authMethod, reason).inc(); } + + /** + * Log authenticate failure event to the authentication metrics. + * @param providerName The short class name of the provider + * @param authMethod Authentication method name. + * @param errorCode Error code. + */ + public static void authenticateFailure(String providerName, String authMethod, Enum errorCode) { + authFailuresMetrics.labels(providerName, authMethod, errorCode.name()).inc(); + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index bf141e10aa1b9..7a23e56fe0b41 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1379,15 +1379,12 @@ public void testAuthMetrics() throws IOException, AuthenticationException { conf.setProperties(properties); provider.initialize(conf); - String authExceptionMessage = ""; - try { provider.authenticate(new AuthenticationDataSource() { }); fail("Should have failed"); } catch (AuthenticationException e) { // expected, no credential passed - authExceptionMessage = e.getMessage(); } String token = AuthTokenUtils.createToken(secretKey, "subject", Optional.empty()); @@ -1424,7 +1421,8 @@ public String getCommandData() { boolean haveFailed = false; for (Metric metric : cm) { if (Objects.equals(metric.tags.get("auth_method"), "token") - && Objects.equals(metric.tags.get("reason"), authExceptionMessage) + && Objects.equals(metric.tags.get("reason"), + AuthenticationProviderToken.ErrorCode.INVALID_AUTH_DATA.name()) && Objects.equals(metric.tags.get("provider_name"), provider.getClass().getSimpleName())) { haveFailed = true; } From 10019391ea7491965d12371d8f614eb0783d7362 Mon Sep 17 00:00:00 2001 From: jack zhang Date: Fri, 5 May 2023 21:25:14 +0800 Subject: [PATCH 066/494] [fix][broker] Fix Return value of getPartitionedStats doesn't contain subscription type (#20210) (cherry picked from commit 4ac117ffc86111e7c38ddadbf933c8be0749f507) --- .../pulsar/broker/admin/AdminApiTest.java | 18 ++++++++++++++++++ .../data/stats/SubscriptionStatsImpl.java | 2 ++ 2 files changed, 20 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index e33108203cc81..f0ac63f08ae4a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -1266,6 +1266,24 @@ public void testGetStats() throws Exception { assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 0); } + @Test + public void testGetPartitionedStatsContainSubscriptionType() throws Exception { + final String topic = "persistent://prop-xyz/ns1/my-topic" + UUID.randomUUID(); + final int numPartitions = 4; + admin.topics().createPartitionedTopic(topic, numPartitions); + + // create consumer and subscription + final String subName = "my-sub"; + @Cleanup Consumer exclusiveConsumer = pulsarClient.newConsumer().topic(topic) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Exclusive) + .subscribe(); + + TopicStats topicStats = admin.topics().getPartitionedStats(topic, false); + assertEquals(topicStats.getSubscriptions().size(), 1); + assertEquals(topicStats.getSubscriptions().get(subName).getType(), SubscriptionType.Exclusive.toString()); + } + @Test public void testGetPartitionedStatsInternal() throws Exception { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index 7ae439829b319..ea7639a8cd2c6 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -166,6 +166,7 @@ public void reset() { backlogSize = 0; msgBacklogNoDelayed = 0; unackedMessages = 0; + type = null; msgRateExpired = 0; totalMsgExpired = 0; lastExpireTimestamp = 0L; @@ -199,6 +200,7 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) { this.msgBacklogNoDelayed += stats.msgBacklogNoDelayed; this.msgDelayed += stats.msgDelayed; this.unackedMessages += stats.unackedMessages; + this.type = stats.type; this.msgRateExpired += stats.msgRateExpired; this.totalMsgExpired += stats.totalMsgExpired; this.isReplicated |= stats.isReplicated; From d1348522d82f7fd972b26af6fd23e55049b0943e Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 6 May 2023 13:46:05 +0800 Subject: [PATCH 067/494] [fix] [broker] Producer created by replicator is not displayed in topic stats (#20229) ### Motivation A producer of the remote cluster is automatically created when replication is turned on. But we can't see anything about it from the response of `(remote cluster) pulsar-admin topic stats` ### Modifications Make this producer displayed in the topic stats (cherry picked from commit 9f7a539593de57ad8c4d224fde81a9e04ac38494) --- .../service/persistent/PersistentTopic.java | 3 +- .../broker/service/OneWayReplicatorTest.java | 78 +++++++ .../service/OneWayReplicatorTestBase.java | 219 ++++++++++++++++++ 3 files changed, 298 insertions(+), 2 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index fe181bb1c01f2..15854f55c5cd1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2226,9 +2226,8 @@ public CompletableFuture asyncGetStats(boolean getPreciseBacklog if (producer.isRemote()) { remotePublishersStats.put(producer.getRemoteCluster(), publisherStats); - } else { - stats.addPublisher(publisherStats); } + stats.addPublisher(publisherStats); }); stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java new file mode 100644 index 0000000000000..e8a21502fb1af --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.common.policies.data.TopicStats; +import org.junit.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class OneWayReplicatorTest extends OneWayReplicatorTestBase { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testReplicatorProducerStatInTopic() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + final String subscribeName = "subscribe_1"; + final byte[] msgValue = "test".getBytes(); + + admin1.topics().createNonPartitionedTopic(topicName); + admin2.topics().createNonPartitionedTopic(topicName); + admin1.topics().createSubscription(topicName, subscribeName, MessageId.earliest); + admin2.topics().createSubscription(topicName, subscribeName, MessageId.earliest); + + // Verify replicator works. + Producer producer1 = client1.newProducer().topic(topicName).create(); + Consumer consumer2 = client2.newConsumer().topic(topicName).subscriptionName(subscribeName).subscribe(); + producer1.newMessage().value(msgValue).send(); + pulsar1.getBrokerService().checkReplicationPolicies(); + assertEquals(consumer2.receive(10, TimeUnit.SECONDS).getValue(), msgValue); + + // Verify there has one item in the attribute "publishers" or "replications" + TopicStats topicStats2 = admin2.topics().getStats(topicName); + Assert.assertTrue(topicStats2.getPublishers().size() + topicStats2.getReplication().size() > 0); + + // cleanup. + consumer2.close(); + producer1.close(); + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java new file mode 100644 index 0000000000000..33620716288af --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTestBase.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Sets; +import java.net.URL; +import java.util.Collections; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; + +@Slf4j +public abstract class OneWayReplicatorTestBase extends TestRetrySupport { + + protected final String defaultTenant = "public"; + protected final String defaultNamespace = defaultTenant + "/default"; + + protected final String cluster1 = "r1"; + protected URL url1; + protected URL urlTls1; + protected ServiceConfiguration config1 = new ServiceConfiguration(); + protected ZookeeperServerTest brokerConfigZk1; + protected LocalBookkeeperEnsemble bkEnsemble1; + protected PulsarService pulsar1; + protected BrokerService ns1; + protected PulsarAdmin admin1; + protected PulsarClient client1; + + protected URL url2; + protected URL urlTls2; + protected final String cluster2 = "r2"; + protected ServiceConfiguration config2 = new ServiceConfiguration(); + protected ZookeeperServerTest brokerConfigZk2; + protected LocalBookkeeperEnsemble bkEnsemble2; + protected PulsarService pulsar2; + protected BrokerService ns2; + protected PulsarAdmin admin2; + protected PulsarClient client2; + + protected void startZKAndBK() throws Exception { + // Start ZK. + brokerConfigZk1 = new ZookeeperServerTest(0); + brokerConfigZk1.start(); + brokerConfigZk2 = new ZookeeperServerTest(0); + brokerConfigZk2.start(); + + // Start BK. + bkEnsemble1 = new LocalBookkeeperEnsemble(3, 0, () -> 0); + bkEnsemble1.start(); + bkEnsemble2 = new LocalBookkeeperEnsemble(3, 0, () -> 0); + bkEnsemble2.start(); + } + + protected void startBrokers() throws Exception { + // Start brokers. + setConfigDefaults(config1, cluster1, bkEnsemble1, brokerConfigZk1); + pulsar1 = new PulsarService(config1); + pulsar1.start(); + ns1 = pulsar1.getBrokerService(); + + url1 = new URL(pulsar1.getWebServiceAddress()); + urlTls1 = new URL(pulsar1.getWebServiceAddressTls()); + admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); + client1 = PulsarClient.builder().serviceUrl(url1.toString()).build(); + + // Start region 2 + setConfigDefaults(config2, cluster2, bkEnsemble2, brokerConfigZk2); + pulsar2 = new PulsarService(config2); + pulsar2.start(); + ns2 = pulsar2.getBrokerService(); + + url2 = new URL(pulsar2.getWebServiceAddress()); + urlTls2 = new URL(pulsar2.getWebServiceAddressTls()); + admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); + client2 = PulsarClient.builder().serviceUrl(url2.toString()).build(); + } + + protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { + admin1.clusters().createCluster(cluster1, ClusterData.builder() + .serviceUrl(url1.toString()) + .serviceUrlTls(urlTls1.toString()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin1.clusters().createCluster(cluster2, ClusterData.builder() + .serviceUrl(url2.toString()) + .serviceUrlTls(urlTls2.toString()) + .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin2.clusters().createCluster(cluster1, ClusterData.builder() + .serviceUrl(url1.toString()) + .serviceUrlTls(urlTls1.toString()) + .brokerServiceUrl(pulsar1.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar1.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + admin2.clusters().createCluster(cluster2, ClusterData.builder() + .serviceUrl(url2.toString()) + .serviceUrlTls(urlTls2.toString()) + .brokerServiceUrl(pulsar2.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar2.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + + admin1.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(cluster1, cluster2))); + admin2.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(cluster1, cluster2))); + + admin1.namespaces().createNamespace(defaultNamespace, Sets.newHashSet(cluster1, cluster2)); + admin2.namespaces().createNamespace(defaultNamespace); + } + + protected void cleanupTopics(CleanupTopicAction cleanupTopicAction) throws Exception { + admin1.namespaces().setNamespaceReplicationClusters(defaultNamespace, Collections.singleton(cluster1)); + admin1.namespaces().unload(defaultNamespace); + cleanupTopicAction.run(); + admin1.namespaces().setNamespaceReplicationClusters(defaultNamespace, Sets.newHashSet(cluster1, cluster2)); + } + + protected interface CleanupTopicAction { + void run() throws Exception; + } + + @Override + protected void setup() throws Exception { + incrementSetupNumber(); + + log.info("--- Starting OneWayReplicatorTestBase::setup ---"); + + startZKAndBK(); + + startBrokers(); + + createDefaultTenantsAndClustersAndNamespace(); + + Thread.sleep(100); + log.info("--- OneWayReplicatorTestBase::setup completed ---"); + } + + private void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setWebServicePort(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bookkeeperEnsemble.getZookeeperPort()); + config.setConfigurationMetadataStoreUrl("zk:127.0.0.1:" + brokerConfigZk.getZookeeperPort() + "/foo"); + config.setBrokerDeleteInactiveTopicsEnabled(false); + config.setBrokerDeleteInactiveTopicsFrequencySeconds(60); + config.setBrokerShutdownTimeoutMs(0L); + config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); + config.setBacklogQuotaCheckIntervalInSeconds(5); + config.setDefaultNumberOfNamespaceBundles(1); + config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); + config.setEnableReplicatedSubscriptions(true); + config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + } + + @Override + protected void cleanup() throws Exception { + markCurrentSetupNumberCleaned(); + log.info("--- Shutting down ---"); + + // Stop brokers. + client1.close(); + client2.close(); + admin1.close(); + admin2.close(); + if (pulsar2 != null) { + pulsar2.close(); + } + if (pulsar1 != null) { + pulsar1.close(); + } + + // Stop ZK and BK. + bkEnsemble1.stop(); + bkEnsemble2.stop(); + brokerConfigZk1.stop(); + brokerConfigZk2.stop(); + + // Reset configs. + config1 = new ServiceConfiguration(); + setConfigDefaults(config1, cluster1, bkEnsemble1, brokerConfigZk1); + config2 = new ServiceConfiguration(); + setConfigDefaults(config2, cluster2, bkEnsemble2, brokerConfigZk2); + } +} From 20c2b5bd653c9c738279f2bd41a7eaa2da797b8f Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 7 May 2023 14:21:51 +0800 Subject: [PATCH 068/494] [fix] [broker] Fix infinite ack of Replicator after topic is closed (#20232) (cherry picked from commit 98413642995eb6e562f6a591dcf56e20ac0cc7ef) --- .../persistent/PersistentReplicator.java | 7 +++ .../pulsar/broker/service/ReplicatorTest.java | 45 +++++++++++++------ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index a556237f4342c..d882cbf56b2e8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -545,6 +545,13 @@ public void deleteComplete(Object ctx) { public void deleteFailed(ManagedLedgerException exception, Object ctx) { log.error("[{}] Failed to delete message at {}: {}", replicatorId, ctx, exception.getMessage(), exception); + if (exception instanceof CursorAlreadyClosedException) { + log.error("[{}] Asynchronous ack failure because replicator is already deleted and cursor is already" + + " closed {}, ({})", replicatorId, ctx, exception.getMessage(), exception); + // replicator is already deleted and cursor is already closed so, producer should also be stopped + closeProducerAsync(); + return; + } if (ctx instanceof PositionImpl) { PositionImpl deletedEntry = (PositionImpl) ctx; if (deletedEntry.compareTo((PositionImpl) cursor.getMarkDeletedPosition()) > 0) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 901451c022bfe..176eab0e94b3d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -21,6 +21,7 @@ import static org.apache.pulsar.broker.BrokerTestUtil.newUniqueName; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; @@ -51,8 +52,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -60,7 +63,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.CursorAlreadyClosedException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; -import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.State; import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; @@ -1645,20 +1647,41 @@ public void testReplicatorWithFailedAck() throws Exception { log.info("--- Starting ReplicatorTest::testReplication ---"); String namespace = "pulsar/global/ns2"; - admin1.namespaces().createNamespace(namespace); - admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + admin1.namespaces().createNamespace(namespace, Sets.newHashSet("r1")); final TopicName dest = TopicName .get(BrokerTestUtil.newUniqueName("persistent://" + namespace + "/ackFailedTopic")); @Cleanup MessageProducer producer1 = new MessageProducer(url1, dest); - log.info("--- Starting producer --- " + url1); + PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(dest.toString(), false) + .getNow(null).get(); + final ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topic.getManagedLedger(); + final ManagedCursorImpl cursor = (ManagedCursorImpl) managedLedger.openCursor("pulsar.repl.r2"); + final ManagedCursorImpl spyCursor = spy(cursor); + managedLedger.getCursors().removeCursor(cursor.getName()); + managedLedger.getCursors().add(spyCursor, PositionImpl.EARLIEST); + AtomicBoolean isMakeAckFail = new AtomicBoolean(false); + doAnswer(invocation -> { + Position pos = (Position) invocation.getArguments()[0]; + AsyncCallbacks.DeleteCallback cb = (AsyncCallbacks.DeleteCallback) invocation.getArguments()[1]; + Object ctx = invocation.getArguments()[2]; + if (isMakeAckFail.get()) { + log.info("async-delete {} will be failed", pos); + cb.deleteFailed(new ManagedLedgerException("mocked error"), ctx); + } else { + log.info("async-delete {} will success", pos); + cursor.asyncDelete(pos, cb, ctx); + } + return null; + }).when(spyCursor).asyncDelete(Mockito.any(Position.class), Mockito.any(AsyncCallbacks.DeleteCallback.class), + Mockito.any()); + + log.info("--- Starting producer --- " + url1); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); // Produce from cluster1 and consume from the rest producer1.produce(2); - PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(dest.toString(), false) - .getNow(null).get(); MessageIdImpl lastMessageId = (MessageIdImpl) topic.getLastMessageId().get(); Position lastPosition = PositionImpl.get(lastMessageId.getLedgerId(), lastMessageId.getEntryId()); ConcurrentOpenHashMap replicators = topic.getReplicators(); @@ -1667,25 +1690,19 @@ public void testReplicatorWithFailedAck() throws Exception { Awaitility.await().pollInterval(1, TimeUnit.SECONDS).timeout(30, TimeUnit.SECONDS) .untilAsserted(() -> assertEquals(org.apache.pulsar.broker.service.AbstractReplicator.State.Started, replicator.getState())); - assertEquals(replicator.getState(), org.apache.pulsar.broker.service.AbstractReplicator.State.Started); - ManagedCursorImpl cursor = (ManagedCursorImpl) replicator.getCursor(); // Make sure all the data has replicated to the remote cluster before close the cursor. Awaitility.await().untilAsserted(() -> assertEquals(cursor.getMarkDeletedPosition(), lastPosition)); - cursor.setState(State.Closed); - - Field field = ManagedCursorImpl.class.getDeclaredField("state"); - field.setAccessible(true); - field.set(cursor, State.Closed); + isMakeAckFail.set(true); producer1.produce(10); // The cursor is closed, so the mark delete position will not move forward. assertEquals(cursor.getMarkDeletedPosition(), lastPosition); - field.set(cursor, State.Open); + isMakeAckFail.set(false); Awaitility.await().timeout(30, TimeUnit.SECONDS).until( () -> { From c8ca9129ff22bf48d4bdcd86e61697382c7e344a Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 8 May 2023 16:23:30 +0800 Subject: [PATCH 069/494] [fix][broker] Fix `RoaringBitmap.contains` can't check value 65535 (#20176) (cherry picked from commit 2f9f5df4a2b4d33ba4a0c7b3ca19267faa50d399) --- .../server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- .../org/apache/pulsar/RoaringbitmapTest.java | 56 +++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/RoaringbitmapTest.java diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 3a30e86904dce..e10c6018ccef8 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -496,7 +496,7 @@ The Apache Software License, Version 2.0 * RabbitMQ Java Client - com.rabbitmq-amqp-client-5.5.3.jar * RoaringBitmap - - org.roaringbitmap-RoaringBitmap-0.9.15.jar + - org.roaringbitmap-RoaringBitmap-0.9.44.jar BSD 3-clause "New" or "Revised" License * Google auth library diff --git a/pom.xml b/pom.xml index 252cedbf65927..c949d0b10618c 100644 --- a/pom.xml +++ b/pom.xml @@ -298,7 +298,7 @@ flexible messaging model and an intuitive client API. 1.3 0.4 8.1.2 - 0.9.15 + 0.9.44 1.6.1 6.4.0 diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/RoaringbitmapTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/RoaringbitmapTest.java new file mode 100644 index 0000000000000..477e7413ef991 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/RoaringbitmapTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar; + +import static org.testng.Assert.assertTrue; +import org.roaringbitmap.RoaringBitmap; +import org.testng.annotations.Test; + +public class RoaringbitmapTest { + + @Test + public void testRoaringBitmapContains() { + RoaringBitmap roaringBitmap = new RoaringBitmap(); + for (long i = 1; i <= 100_000; i++) { + roaringBitmap.add(i, i + 1); + } + + for (long i = 1; i <= 100_000; i++) { + assertTrue(roaringBitmap.contains(i, i + 1)); + } + + RoaringBitmap roaringBitmap2 = new RoaringBitmap(); + for (long i = 1; i <= 1000_000; i++) { + roaringBitmap2.add(i, i + 1); + } + + for (long i = 1; i <= 1000_000; i++) { + assertTrue(roaringBitmap2.contains(i, i + 1)); + } + + RoaringBitmap roaringBitmap3 = new RoaringBitmap(); + for (long i = 1; i <= 10_000_000; i++) { + roaringBitmap3.add(i, i + 1); + } + + for (long i = 1; i <= 10_000_000; i++) { + assertTrue(roaringBitmap3.contains(i, i + 1)); + } + } +} From bfe08a9a4ac893a96cadfa51567f6b696e021324 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Tue, 9 May 2023 10:06:36 +0800 Subject: [PATCH 070/494] [improve][ci] Enable CI tests for PRs (#20254) --- .github/workflows/ci-go-functions.yaml | 1 + .github/workflows/pulsar-ci-flaky.yaml | 1 + .github/workflows/pulsar-ci.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index a34cb1a90908e..1ff0c8eef57bf 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-3.0 paths: - '.github/workflows/**' - 'pulsar-function-go/**' diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index ea19bc3307819..198823c3008d1 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-3.0 schedule: - cron: '0 12 * * *' workflow_dispatch: diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 4d4f2902d8f27..3e2a67e3cf070 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-3.0 schedule: - cron: '0 12 * * *' workflow_dispatch: From c6c1390a99e66a1f79c7b0d9d07e94512a05ff95 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Tue, 9 May 2023 14:02:51 +0800 Subject: [PATCH 071/494] [fix][broker] Fix the behavior of delayed message in Key_Shared mode (#20233) (cherry picked from commit 00f17e88ad8fa9a4ead78d78a181c03fdee8e4df) --- ...PersistentDispatcherMultipleConsumers.java | 9 ++- .../client/api/KeySharedSubscriptionTest.java | 76 +++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java index 81adda053e8cd..b3d48252efe58 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherMultipleConsumers.java @@ -1070,14 +1070,15 @@ public boolean trackDelayedDelivery(long ledgerId, long entryId, MessageMetadata } protected synchronized NavigableSet getMessagesToReplayNow(int maxMessagesToRead) { - if (!redeliveryMessages.isEmpty()) { - return redeliveryMessages.getMessagesToReplayNow(maxMessagesToRead); - } else if (delayedDeliveryTracker.isPresent() && delayedDeliveryTracker.get().hasMessageAvailable()) { + if (delayedDeliveryTracker.isPresent() && delayedDeliveryTracker.get().hasMessageAvailable()) { delayedDeliveryTracker.get().resetTickTime(topic.getDelayedDeliveryTickTimeMillis()); NavigableSet messagesAvailableNow = delayedDeliveryTracker.get().getScheduledMessages(maxMessagesToRead); messagesAvailableNow.forEach(p -> redeliveryMessages.add(p.getLedgerId(), p.getEntryId())); - return messagesAvailableNow; + } + + if (!redeliveryMessages.isEmpty()) { + return redeliveryMessages.getMessagesToReplayNow(maxMessagesToRead); } else { return Collections.emptyNavigableSet(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java index 7b617a6a192d8..18fb141be3178 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java @@ -114,6 +114,7 @@ public Object[][] topicDomainProvider() { @BeforeClass(alwaysRun = true) @Override protected void setup() throws Exception { + this.conf.setUnblockStuckSubscriptionEnabled(true); super.internalSetup(); super.producerBaseSetup(); this.conf.setSubscriptionKeySharedUseConsistentHashing(true); @@ -1554,4 +1555,79 @@ public void testStickyKeyRangesRestartConsumers() throws Exception { producerFuture.get(); } + + @Test + public void testContinueDispatchMessagesWhenMessageDelayed() throws Exception { + int delayedMessages = 40; + int messages = 40; + int sum = 0; + final String topic = "persistent://public/default/key_shared-" + UUID.randomUUID(); + final String subName = "my-sub"; + + @Cleanup + Consumer consumer1 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(10) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topic) + .create(); + + for (int i = 0; i < delayedMessages; i++) { + MessageId messageId = producer.newMessage() + .key(String.valueOf(random.nextInt(NUMBER_OF_KEYS))) + .value(100 + i) + .deliverAfter(10, TimeUnit.SECONDS) + .send(); + log.info("Published delayed message :{}", messageId); + } + + for (int i = 0; i < messages; i++) { + MessageId messageId = producer.newMessage() + .key(String.valueOf(random.nextInt(NUMBER_OF_KEYS))) + .value(i) + .send(); + log.info("Published message :{}", messageId); + } + + @Cleanup + Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName(subName) + .receiverQueueSize(30) + .subscriptionType(SubscriptionType.Key_Shared) + .subscribe(); + + for (int i = 0; i < delayedMessages + messages; i++) { + Message msg = consumer1.receive(30, TimeUnit.SECONDS); + if (msg != null) { + log.info("c1 message: {}, {}", msg.getValue(), msg.getMessageId()); + consumer1.acknowledge(msg); + } else { + break; + } + sum++; + } + + log.info("Got {} messages...", sum); + + int remaining = delayedMessages + messages - sum; + for (int i = 0; i < remaining; i++) { + Message msg = consumer2.receive(30, TimeUnit.SECONDS); + if (msg != null) { + log.info("c2 message: {}, {}", msg.getValue(), msg.getMessageId()); + consumer2.acknowledge(msg); + } else { + break; + } + sum++; + } + + log.info("Got {} other messages...", sum); + Assert.assertEquals(sum, delayedMessages + messages); + } } From e42faff4a9f9c88a6631b1348cf60ae024dade5c Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 9 May 2023 23:27:34 +0800 Subject: [PATCH 072/494] [improve][build] Upgrade dependency to fix CVE. (#20264) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 32 ++++++------- .../shell/src/assemble/LICENSE.bin.txt | 46 +++++++++---------- pom.xml | 6 +-- pulsar-broker-auth-oidc/pom.xml | 13 ++++++ pulsar-functions/runtime/pom.xml | 14 ++++++ .../KubernetesSecretsTokenAuthProvider.java | 10 ++-- .../runtime/kubernetes/KubernetesRuntime.java | 11 ++--- .../kubernetes/KubernetesRuntimeFactory.java | 2 +- ...ubernetesSecretsTokenAuthProviderTest.java | 2 +- .../KubernetesRuntimeFactoryTest.java | 6 +-- pulsar-functions/secrets/pom.xml | 14 ++++++ pulsar-io/flume/pom.xml | 12 +++++ pulsar-io/kafka-connect-adaptor/pom.xml | 7 +++ pulsar-sql/presto-distribution/LICENSE | 30 ++++++------ src/owasp-dependency-check-suppressions.xml | 8 ---- 16 files changed, 133 insertions(+), 82 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 704a221155854..695ae694001e6 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -52,7 +52,7 @@ 4.2.3 31.0.1-jre 1.10.12 - 1.32 + 2.0 3.12.4 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index e10c6018ccef8..de10128ec3686 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -246,21 +246,21 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.82.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.13.4.jar - - com.fasterxml.jackson.core-jackson-core-2.13.4.jar - - com.fasterxml.jackson.core-jackson-databind-2.13.4.2.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.4.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.4.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.4.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.4.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.4.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.13.4.jar - - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.13.4.jar - - com.fasterxml.jackson.module-jackson-module-parameter-names-2.13.4.jar + - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar + - com.fasterxml.jackson.core-jackson-core-2.14.2.jar + - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar + - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar + - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar - * Bitbucket -- org.bitbucket.b_c-jose4j-0.7.6.jar + * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.3.jar * Gson - com.google.code.gson-gson-2.8.9.jar - io.gsonfire-gson-fire-1.8.5.jar @@ -402,7 +402,7 @@ The Apache Software License, Version 2.0 - org.eclipse.jetty.websocket-websocket-servlet-9.4.51.v20230217.jar - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.51.v20230217.jar - org.eclipse.jetty-jetty-alpn-server-9.4.51.v20230217.jar - * SnakeYaml -- org.yaml-snakeyaml-1.32.jar + * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar @@ -453,9 +453,9 @@ The Apache Software License, Version 2.0 * Apache Yetus - org.apache.yetus-audience-annotations-0.12.0.jar * Kubernetes Client - - io.kubernetes-client-java-12.0.1.jar - - io.kubernetes-client-java-api-12.0.1.jar - - io.kubernetes-client-java-proto-12.0.1.jar + - io.kubernetes-client-java-18.0.0.jar + - io.kubernetes-client-java-api-18.0.0.jar + - io.kubernetes-client-java-proto-18.0.0.jar * Dropwizard - io.dropwizard.metrics-metrics-core-4.1.12.1.jar - io.dropwizard.metrics-metrics-graphite-4.1.12.1.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index cf741622c756f..3021df8c63dee 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -311,17 +311,17 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * JCommander -- jcommander-1.82.jar * Jackson - - jackson-annotations-2.13.4.jar - - jackson-core-2.13.4.jar - - jackson-databind-2.13.4.2.jar - - jackson-dataformat-yaml-2.13.4.jar - - jackson-jaxrs-base-2.13.4.jar - - jackson-jaxrs-json-provider-2.13.4.jar - - jackson-module-jaxb-annotations-2.13.4.jar - - jackson-module-jsonSchema-2.13.4.jar - - jackson-datatype-jdk8-2.13.4.jar - - jackson-datatype-jsr310-2.13.4.jar - - jackson-module-parameter-names-2.13.4.jar + - jackson-annotations-2.14.2.jar + - jackson-core-2.14.2.jar + - jackson-databind-2.14.2.jar + - jackson-dataformat-yaml-2.14.2.jar + - jackson-jaxrs-base-2.14.2.jar + - jackson-jaxrs-json-provider-2.14.2.jar + - jackson-module-jaxb-annotations-2.14.2.jar + - jackson-module-jsonSchema-2.14.2.jar + - jackson-datatype-jdk8-2.14.2.jar + - jackson-datatype-jsr310-2.14.2.jar + - jackson-module-parameter-names-2.14.2.jar * Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar * Gson - gson-2.8.9.jar @@ -332,9 +332,9 @@ The Apache Software License, Version 2.0 * J2ObjC Annotations -- j2objc-annotations-1.3.jar * Netty Reactive Streams -- netty-reactive-streams-2.0.6.jar * Swagger - - swagger-annotations-1.6.2.jar - - swagger-core-1.6.2.jar - - swagger-models-1.6.2.jar + - swagger-annotations-1.6.10.jar + - swagger-core-1.6.10.jar + - swagger-models-1.6.10.jar * DataSketches - memory-0.8.3.jar - sketches-core-0.8.3.jar @@ -399,15 +399,15 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Jetty - - jetty-client-9.4.48.v20220622.jar - - jetty-http-9.4.48.v20220622.jar - - jetty-io-9.4.48.v20220622.jar - - jetty-util-9.4.48.v20220622.jar - - javax-websocket-client-impl-9.4.48.v20220622.jar - - websocket-api-9.4.48.v20220622.jar - - websocket-client-9.4.48.v20220622.jar - - websocket-common-9.4.48.v20220622.jar - * SnakeYaml -- snakeyaml-1.32.jar + - jetty-client-9.4.51.v20230217.jar + - jetty-http-9.4.51.v20230217.jar + - jetty-io-9.4.51.v20230217.jar + - jetty-util-9.4.51.v20230217.jar + - javax-websocket-client-impl-9.4.51.v20230217.jar + - websocket-api-9.4.51.v20230217.jar + - websocket-client-9.4.51.v20230217.jar + - websocket-common-9.4.51.v20230217.jar + * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar * Javassist -- javassist-3.25.0-GA.jar * Apache Avro diff --git a/pom.xml b/pom.xml index c949d0b10618c..84c8977f51968 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ flexible messaging model and an intuitive client API. 1.69 1.0.6 1.0.2.3 - 2.13.4.20221013 + 2.14.2 0.10.2 1.6.10 8.37 @@ -230,7 +230,7 @@ flexible messaging model and an intuitive client API. 2.3.3 2.0.2 5.12.1 - 12.0.1 + 18.0.0 4.9.3 2.8.0 @@ -242,7 +242,7 @@ flexible messaging model and an intuitive client API. 4.5.13 4.4.15 0.5.11 - 1.32 + 2.0 1.10.12 5.3.3 3.4.3 diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index 6eaa665394504..bb50786202301 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -83,7 +83,20 @@ io.prometheus simpleclient_httpserver + + bcpkix-jdk18on + org.bouncycastle + + + bcutil-jdk18on + org.bouncycastle + + + bcprov-jdk18on + org.bouncycastle + + diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 1789c85f62be5..689cdac84add2 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -64,6 +64,20 @@ io.kubernetes client-java ${kubernetesclient.version} + + + bcpkix-jdk18on + org.bouncycastle + + + bcutil-jdk18on + org.bouncycastle + + + bcprov-jdk18on + org.bouncycastle + + ${project.groupId} diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java index 1053e6e170ef1..916b8e9a6e1ad 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java @@ -205,8 +205,7 @@ public void cleanUpAuthData(Function.FunctionDetails funcDetails, Optional { try { - coreClient.readNamespacedSecret(secretName, kubeNamespace, - null, null, null); + coreClient.readNamespacedSecret(secretName, kubeNamespace, null); } catch (ApiException e) { // statefulset is gone @@ -305,12 +304,13 @@ private void upsertSecret(String token, Function.FunctionDetails funcDetails, St .data(buildSecretMap(token)); try { - coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null); + coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null, null); } catch (ApiException e) { if (e.getCode() == HTTP_CONFLICT) { try { coreClient - .replaceNamespacedSecret(secretName, kubeNamespace, v1Secret, null, null, null); + .replaceNamespacedSecret(secretName, kubeNamespace, + v1Secret, null, null, null, null); return Actions.ActionResult.builder().success(true).build(); } catch (ApiException e1) { @@ -366,7 +366,7 @@ private String createSecret(String token, Function.FunctionDetails funcDetails) .metadata(new V1ObjectMeta().name(getSecretName(id))) .data(buildSecretMap(token)); try { - coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null); + coreClient.createNamespacedSecret(kubeNamespace, v1Secret, null, null, null, null); } catch (ApiException e) { // already exists if (e.getCode() == HTTP_CONFLICT) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index b1df6c098f6e3..939a446d7fed7 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -472,7 +472,7 @@ private void submitService() throws Exception { .supplier(() -> { final V1Service response; try { - response = coreClient.createNamespacedService(jobNamespace, service, null, null, null); + response = coreClient.createNamespacedService(jobNamespace, service, null, null, null, null); } catch (ApiException e) { // already exists if (e.getCode() == HTTP_CONFLICT) { @@ -561,7 +561,8 @@ private void submitStatefulSet() throws Exception { .supplier(() -> { final V1StatefulSet response; try { - response = appsClient.createNamespacedStatefulSet(jobNamespace, statefulSet, null, null, null); + response = appsClient.createNamespacedStatefulSet(jobNamespace, + statefulSet, null, null, null, null); } catch (ApiException e) { // already exists if (e.getCode() == HTTP_CONFLICT) { @@ -657,8 +658,7 @@ public void deleteStatefulSet() throws InterruptedException { .supplier(() -> { V1StatefulSet response; try { - response = appsClient.readNamespacedStatefulSet(statefulSetName, jobNamespace, - null, null, null); + response = appsClient.readNamespacedStatefulSet(statefulSetName, jobNamespace, null); } catch (ApiException e) { // statefulset is gone if (e.getCode() == HTTP_NOT_FOUND) { @@ -805,8 +805,7 @@ public void deleteService() throws InterruptedException { .supplier(() -> { V1Service response; try { - response = coreClient.readNamespacedService(serviceName, jobNamespace, - null, null, null); + response = coreClient.readNamespacedService(serviceName, jobNamespace, null); } catch (ApiException e) { // service is gone diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index 895304138a530..3e1d40e80dc6e 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -405,7 +405,7 @@ static void fetchConfigMap(CoreV1Api coreClient, String changeConfigMap, KubernetesRuntimeFactory kubernetesRuntimeFactory) { try { V1ConfigMap v1ConfigMap = - coreClient.readNamespacedConfigMap(changeConfigMap, changeConfigMapNamespace, null, true, false); + coreClient.readNamespacedConfigMap(changeConfigMap, changeConfigMapNamespace, null); Map data = v1ConfigMap.getData(); if (data != null) { overRideKubernetesConfig(data, kubernetesRuntimeFactory); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java index 081e693b6a321..cf294afcf9b78 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProviderTest.java @@ -103,7 +103,7 @@ public void testConfigureAuthDataStatefulSetNoCa() { @Test public void testCacheAuthData() throws ApiException { CoreV1Api coreV1Api = mock(CoreV1Api.class); - doReturn(new V1Secret()).when(coreV1Api).createNamespacedSecret(anyString(), any(), anyString(), anyString(), anyString()); + doReturn(new V1Secret()).when(coreV1Api).createNamespacedSecret(anyString(), any(), anyString(), anyString(), anyString(), anyString()); KubernetesSecretsTokenAuthProvider kubernetesSecretsTokenAuthProvider = new KubernetesSecretsTokenAuthProvider(); kubernetesSecretsTokenAuthProvider.initialize(coreV1Api, null, (fd) -> "default"); Function.FunctionDetails funcDetails = Function.FunctionDetails.newBuilder().setTenant("test-tenant").setNamespace("test-ns").setName("test-func").build(); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java index a5fc8f231a6b5..48497bf218d40 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactoryTest.java @@ -468,9 +468,9 @@ public void testDynamicConfigMapLoading() throws Exception { KubernetesRuntimeFactory kubernetesRuntimeFactory = getKuberentesRuntimeFactory(); CoreV1Api coreV1Api = Mockito.mock(CoreV1Api.class); V1ConfigMap v1ConfigMap = new V1ConfigMap(); - Mockito.doReturn(v1ConfigMap).when(coreV1Api).readNamespacedConfigMap(any(), any(), any(), any(), any()); + Mockito.doReturn(v1ConfigMap).when(coreV1Api).readNamespacedConfigMap(any(), any(), any()); KubernetesRuntimeFactory.fetchConfigMap(coreV1Api, changeConfigMap, changeConfigNamespace, kubernetesRuntimeFactory); - Mockito.verify(coreV1Api, Mockito.times(1)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null), eq(true), eq(false)); + Mockito.verify(coreV1Api, Mockito.times(1)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null)); KubernetesRuntimeFactory expected = getKuberentesRuntimeFactory(); assertEquals(kubernetesRuntimeFactory, expected); @@ -479,7 +479,7 @@ public void testDynamicConfigMapLoading() throws Exception { configs.put("imagePullPolicy", "test_imagePullPolicy2"); v1ConfigMap.setData(configs); KubernetesRuntimeFactory.fetchConfigMap(coreV1Api, changeConfigMap, changeConfigNamespace, kubernetesRuntimeFactory); - Mockito.verify(coreV1Api, Mockito.times(2)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null), eq(true), eq(false)); + Mockito.verify(coreV1Api, Mockito.times(2)).readNamespacedConfigMap(eq(changeConfigMap), eq(changeConfigNamespace), eq(null)); assertEquals(kubernetesRuntimeFactory.getPulsarDockerImageName(), "test_dockerImage2"); assertEquals(kubernetesRuntimeFactory.getImagePullPolicy(), "test_imagePullPolicy2"); diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 6bf93d5a03cb7..f2786b8c85bd3 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -35,6 +35,20 @@ io.kubernetes client-java ${kubernetesclient.version} + + + bcpkix-jdk18on + org.bouncycastle + + + bcutil-jdk18on + org.bouncycastle + + + bcprov-jdk18on + org.bouncycastle + + diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 5df8971e11790..1d5afabc98650 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -45,6 +45,10 @@ com.fasterxml.jackson.core jackson-databind + + org.apache.commons + commons-collections4 + com.fasterxml.jackson.dataformat @@ -65,6 +69,10 @@ avro org.apache.avro + + commons-collections + commons-collections + @@ -90,6 +98,10 @@ io.netty netty + + commons-collections + commons-collections + diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index f4f8c9cb3dbe9..b6fc6b428d709 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -147,6 +147,13 @@ test-jar + + org.bouncycastle + bc-fips + ${bouncycastle.bc-fips.version} + test + + org.apache.avro avro diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 9a5e4678baadc..d41155a321d65 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.13.4.jar - - jackson-core-2.13.4.jar - - jackson-databind-2.13.4.2.jar - - jackson-dataformat-smile-2.13.4.jar - - jackson-datatype-guava-2.13.4.jar - - jackson-datatype-jdk8-2.13.4.jar - - jackson-datatype-joda-2.13.4.jar - - jackson-datatype-jsr310-2.13.4.jar - - jackson-dataformat-yaml-2.13.4.jar - - jackson-jaxrs-base-2.13.4.jar - - jackson-jaxrs-json-provider-2.13.4.jar - - jackson-module-jaxb-annotations-2.13.4.jar - - jackson-module-jsonSchema-2.13.4.jar + - jackson-annotations-2.14.2.jar + - jackson-core-2.14.2.jar + - jackson-databind-2.14.2.jar + - jackson-dataformat-smile-2.14.2.jar + - jackson-datatype-guava-2.14.2.jar + - jackson-datatype-jdk8-2.14.2.jar + - jackson-datatype-joda-2.14.2.jar + - jackson-datatype-jsr310-2.14.2.jar + - jackson-dataformat-yaml-2.14.2.jar + - jackson-jaxrs-base-2.14.2.jar + - jackson-jaxrs-json-provider-2.14.2.jar + - jackson-module-jaxb-annotations-2.14.2.jar + - jackson-module-jsonSchema-2.14.2.jar * Guava - guava-31.0.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -401,7 +401,7 @@ The Apache Software License, Version 2.0 * RocksDB JNI - rocksdbjni-7.9.2.jar * SnakeYAML - - snakeyaml-1.32.jar + - snakeyaml-2.0.jar * Bean Validation API - validation-api-2.0.1.Final.jar * Objectsize @@ -456,7 +456,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.8.4.jar * Jackson - - jackson-module-parameter-names-2.13.4.jar + - jackson-module-parameter-names-2.14.2.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 4bca86ab12620..dd95cbc1025ef 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -37,14 +37,6 @@ .* - - - e80612549feb5c9191c498de628c1aa80693cf0b - CVE-2022-1471 - - Date: Wed, 10 May 2023 13:26:16 +0200 Subject: [PATCH 073/494] [fix][monitor] topic with double quote breaks the prometheus format (#20230) (cherry picked from commit ea56197d18d8d1f5b06c26246d920cb736d4e6a6) (cherry picked from commit c5b18542b8cf0806616a3848dbe14ef2394e57df) --- .../prometheus/PrometheusMetricStreams.java | 6 ++++- .../broker/stats/PrometheusMetricsTest.java | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java index df107b6d7d9c4..93cbad4e19503 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricStreams.java @@ -41,7 +41,11 @@ void writeSample(String metricName, Number value, String... labelsAndValuesArray SimpleTextOutputStream stream = initGaugeType(metricName); stream.write(metricName).write('{'); for (int i = 0; i < labelsAndValuesArray.length; i += 2) { - stream.write(labelsAndValuesArray[i]).write("=\"").write(labelsAndValuesArray[i + 1]).write('\"'); + String labelValue = labelsAndValuesArray[i + 1]; + if (labelValue != null) { + labelValue = labelValue.replace("\"", "\\\""); + } + stream.write(labelsAndValuesArray[i]).write("=\"").write(labelValue).write('\"'); if (i + 2 != labelsAndValuesArray.length) { stream.write(','); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java index 7a23e56fe0b41..a7a28afd8ac64 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/PrometheusMetricsTest.java @@ -1953,4 +1953,30 @@ public String toString() { } } + @Test + public void testEscapeLabelValue() throws Exception { + String ns1 = "prop/ns-abc1"; + admin.namespaces().createNamespace(ns1); + String topic = "persistent://" + ns1 + "/\"mytopic"; + admin.topics().createNonPartitionedTopic(topic); + + @Cleanup + final Consumer consumer = pulsarClient.newConsumer() + .subscriptionName("sub") + .topic(topic) + .subscribe(); + @Cleanup + ByteArrayOutputStream statsOut = new ByteArrayOutputStream(); + PrometheusMetricsGenerator.generate(pulsar, true, false, + false, statsOut); + String metricsStr = statsOut.toString(); + final List subCountLines = metricsStr.lines() + .filter(line -> line.startsWith("pulsar_subscriptions_count")) + .collect(Collectors.toList()); + System.out.println(subCountLines); + assertEquals(subCountLines.size(), 1); + assertEquals(subCountLines.get(0), + "pulsar_subscriptions_count{cluster=\"test\",namespace=\"prop/ns-abc1\",topic=\"persistent://prop/ns-abc1/\\\"mytopic\"} 1"); + } + } From 76ed071a7e1dddf1023a40bf3563f3b1200d4b4a Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 17 May 2023 19:20:24 -0500 Subject: [PATCH 074/494] [fix][test] Use delta when comparing doubles in checkLoadReportNicSpeed (#20343) ### Motivation @dave2wave recently encountered this test failure: ``` 20:46:55 [ERROR] Tests run: 2, Failures: 1, Errors: 0, Skipped: 1, Time elapsed: 15.301 s <<< FAILURE! - in org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest 20:46:55 [ERROR] checkLoadReportNicSpeed(org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest) Time elapsed: 0.142 s <<< FAILURE! 20:46:55 java.lang.AssertionError: expected [1.6200000000000004E7] but found [1.62E7] 20:46:55 at org.testng.Assert.fail(Assert.java:99) 20:46:55 at org.testng.Assert.failNotEquals(Assert.java:1037) 20:46:55 at org.testng.Assert.assertEquals(Assert.java:701) 20:46:55 at org.testng.Assert.assertEquals(Assert.java:712) 20:46:55 at org.apache.pulsar.broker.loadbalance.LoadReportNetworkLimitTest.checkLoadReportNicSpeed(LoadReportNetworkLimitTest.java:63) ``` The test failed because we are not providing an acceptable delta for the equality check. It looks like there are other tests where we have strict double or float comparisons. I am going to leave those for now to minimize the diff on this PR. I read through all of the open flaky test issues and none appear to be related to this class of flakiness. ### Modifications * Add a delta to the assertions in the `checkLoadReportNicSpeed` test ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Documentation - [x] `doc-not-needed` (cherry picked from commit b0bb5ae1e8606082a0511c71d36b7c6c53d7086d) --- .../broker/loadbalance/LoadReportNetworkLimitTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java index bec971dfa40e7..ffa2607b6b42e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadReportNetworkLimitTest.java @@ -60,12 +60,12 @@ public void checkLoadReportNicSpeed() throws Exception { LoadManagerReport report = admin.brokerStats().getLoadReport(); if (SystemUtils.IS_OS_LINUX) { - assertEquals(report.getBandwidthIn().limit, usableNicCount * 5.4 * 1000 * 1000); - assertEquals(report.getBandwidthOut().limit, usableNicCount * 5.4 * 1000 * 1000); + assertEquals(report.getBandwidthIn().limit, usableNicCount * 5.4 * 1000 * 1000, 0.0001); + assertEquals(report.getBandwidthOut().limit, usableNicCount * 5.4 * 1000 * 1000, 0.0001); } else { // On non-Linux system we don't report the network usage - assertEquals(report.getBandwidthIn().limit, -1.0); - assertEquals(report.getBandwidthOut().limit, -1.0); + assertEquals(report.getBandwidthIn().limit, -1.0, 0.0001); + assertEquals(report.getBandwidthOut().limit, -1.0, 0.0001); } } From 127ea2576d65d72b7906921178565b1a3a9e722e Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 16 May 2023 09:41:35 +0800 Subject: [PATCH 075/494] [fix][txn] Implement compatibility for transaction buffer segmented snapshot feature upgrade (#20235) master https://github.com/apache/pulsar/issues/16913 ## Motivation: The transaction buffer segmented snapshot feature aims to improve the transaction buffer's performance by segmenting the snapshot and managing it more efficiently. However, for existing topics that were created before this feature was introduced, we need to ensure a seamless transition and compatibility when enabling the segmented snapshot feature. ## Modifications: 1. Updated the `recoverFromSnapshot()` method to read from another topic if the `persistentSnapshotIndexes` is null. This ensures that the appropriate snapshot data is fetched during the recovery process when upgrading to the segmented snapshot feature. 2. Created a new test `testSnapshotProcessorUpdate()` that verifies the compatibility of the transaction ### Verifying this change - [ ] Make sure that the change passes the CI checks. *(Please pick either of the following options)* This change is a trivial rework / code cleanup without any test coverage. *(or)* This change is already covered by existing tests, such as *(please describe tests)*. *(or)* This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* (cherry picked from commit 8b929e6bf1b1718431abebf04420d7817ad8cdb7) --- ...napshotSegmentAbortedTxnProcessorImpl.java | 66 ++++++++- .../SegmentAbortedTxnProcessorTest.java | 133 +++++++++++++++++- 2 files changed, 196 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 751c03aff95a9..4f4e58ac3f55b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -47,6 +47,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.metadata.TransactionBufferSnapshot; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndex; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexesMetadata; @@ -265,7 +266,7 @@ public CompletableFuture recoverFromSnapshot() { PositionImpl finalStartReadCursorPosition = startReadCursorPosition; TransactionBufferSnapshotIndexes finalPersistentSnapshotIndexes = persistentSnapshotIndexes; if (persistentSnapshotIndexes == null) { - return CompletableFuture.completedFuture(null); + return recoverOldSnapshot(); } else { this.unsealedTxnIds = convertTypeToTxnID(persistentSnapshotIndexes .getSnapshot().getAborts()); @@ -378,6 +379,69 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob .getExecutor(this)); } + // This method will be deprecated and removed in version 4.x.0 + private CompletableFuture recoverOldSnapshot() { + return topic.getBrokerService().getTopic(TopicName.get(topic.getName()).getNamespace() + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, false) + .thenCompose(topicOption -> { + if (!topicOption.isPresent()) { + return CompletableFuture.completedFuture(null); + } else { + return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() + .getTxnBufferSnapshotService() + .createReader(TopicName.get(topic.getName())).thenComposeAsync(snapshotReader -> { + PositionImpl startReadCursorPositionInOldSnapshot = null; + try { + while (snapshotReader.hasMoreEvents()) { + Message message = snapshotReader.readNextAsync() + .get(getSystemClientOperationTimeoutMs(), TimeUnit.MILLISECONDS); + if (topic.getName().equals(message.getKey())) { + TransactionBufferSnapshot transactionBufferSnapshot = + message.getValue(); + if (transactionBufferSnapshot != null) { + handleOldSnapshot(transactionBufferSnapshot); + startReadCursorPositionInOldSnapshot = PositionImpl.get( + transactionBufferSnapshot.getMaxReadPositionLedgerId(), + transactionBufferSnapshot.getMaxReadPositionEntryId()); + } + } + } + } catch (TimeoutException ex) { + Throwable t = FutureUtil.unwrapCompletionException(ex); + String errorMessage = String.format("[%s] Transaction buffer recover fail by " + + "read transactionBufferSnapshot timeout!", topic.getName()); + log.error(errorMessage, t); + return FutureUtil.failedFuture(new BrokerServiceException + .ServiceUnitNotReadyException(errorMessage, t)); + } catch (Exception ex) { + log.error("[{}] Transaction buffer recover fail when read " + + "transactionBufferSnapshot!", topic.getName(), ex); + return FutureUtil.failedFuture(ex); + } finally { + assert snapshotReader != null; + closeReader(snapshotReader); + } + return CompletableFuture.completedFuture(startReadCursorPositionInOldSnapshot); + }, + topic.getBrokerService().getPulsar().getTransactionExecutorProvider() + .getExecutor(this)); + } + }); + } + + // This method will be deprecated and removed in version 4.x.0 + private void handleOldSnapshot(TransactionBufferSnapshot snapshot) { + if (snapshot.getAborts() != null) { + snapshot.getAborts().forEach(abortTxnMetadata -> { + TxnID txnID = new TxnID(abortTxnMetadata.getTxnIdMostBits(), + abortTxnMetadata.getTxnIdLeastBits()); + aborts.put(txnID, txnID); + //The old data will be written into the first segment. + unsealedTxnIds.add(txnID); + }); + } + } + @Override public CompletableFuture clearAbortedTxnSnapshot() { return persistentWorker.appendTask(PersistentWorker.OperationType.Clear, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index c157d7cf8c527..cb15ab003f7b7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.transaction; +import static org.junit.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -44,10 +45,16 @@ import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.events.EventType; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.TopicStats; import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -247,8 +254,8 @@ public void testClearSnapshotSegments() throws Exception { private void verifySnapshotSegmentsSize(String topic, int size) throws Exception { SystemTopicClient.Reader reader = pulsarService.getTransactionBufferSnapshotServiceFactory() - .getTxnBufferSnapshotSegmentService() - .createReader(TopicName.get(topic)).get(); + .getTxnBufferSnapshotSegmentService() + .createReader(TopicName.get(topic)).get(); int segmentCount = 0; while (reader.hasMoreEvents()) { Message message = reader.readNextAsync() @@ -286,4 +293,126 @@ private void doCompaction(TopicName topic) throws Exception { CompletableFuture compactionFuture = (CompletableFuture) field.get(snapshotTopic); org.awaitility.Awaitility.await().untilAsserted(() -> assertTrue(compactionFuture.isDone())); } + + /** + * This test verifies the compatibility of the transaction buffer segmented snapshot feature + * when enabled on an existing topic. + * It performs the following steps: + * 1. Creates a topic with segmented snapshot disabled. + * 2. Sends 10 messages without using transactions. + * 3. Sends 10 messages using transactions and aborts them. + * 4. Sends 10 messages without using transactions. + * 5. Verifies that only the non-transactional messages are received. + * 6. Enables the segmented snapshot feature and sets the snapshot segment size. + * 7. Unloads the topic. + * 8. Sends a new message using a transaction and aborts it. + * 9. Verifies that the topic has exactly one segment. + * 10. Re-subscribes the consumer and re-verifies that only the non-transactional messages are received. + */ + @Test + public void testSnapshotProcessorUpgrade() throws Exception { + this.pulsarService = getPulsarServiceList().get(0); + this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false); + + // Create a topic, send 10 messages without using transactions, and send 10 messages using transactions. + // Abort these transactions and verify the data. + final String topicName = "persistent://" + NAMESPACE1 + "/testSnapshotProcessorUpgrade"; + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe(); + + // Send 10 messages without using transactions + for (int i = 0; i < 10; i++) { + producer.send(("test-message-" + i).getBytes()); + } + + // Send 10 messages using transactions and abort them + for (int i = 0; i < 10; i++) { + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + producer.newMessage(txn).value(("test-txn-message-" + i).getBytes()).sendAsync(); + txn.abort().get(); + } + + // Send 10 messages without using transactions + for (int i = 10; i < 20; i++) { + producer.send(("test-message-" + i).getBytes()); + } + + // Verify the data + for (int i = 0; i < 20; i++) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals("test-message-" + i, new String(msg.getData())); + } + + // Enable segmented snapshot + this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + + SEGMENT_SIZE * 3); + + // Unload the topic + admin.topics().unload(topicName); + + // Sends a new message using a transaction and aborts it. + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.SECONDS) + .build().get(); + producer.newMessage(txn).value("test-message-new".getBytes()).send(); + txn.abort().get(); + + // Verifies that the topic has exactly one segment. + Awaitility.await().untilAsserted(() -> { + String segmentTopic = "persistent://" + NAMESPACE1 + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS; + TopicStats topicStats = admin.topics().getStats(segmentTopic); + assertEquals(1, topicStats.getMsgInCounter()); + }); + + // Re-subscribes the consumer and re-verifies that only the non-transactional messages are received. + consumer.close(); + consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe(); + for (int i = 0; i < 20; i++) { + Message msg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals("test-message-" + i, new String(msg.getData())); + } + } + + /** + * This test verifies that when the segmented snapshot feature is enabled, creating a new topic + * does not create a __transaction_buffer_snapshot topic in the same namespace. + * The test performs the following steps: + * 1. Enable the segmented snapshot feature. + * 2. Create a new namespace. + * 3. Create a new topic in the namespace. + * 4. Check that the __transaction_buffer_snapshot topic is not created in the same namespace. + * 5. Destroy the namespace after the test. + */ + @Test + public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Exception { + // Enable the segmented snapshot feature + pulsarService = getPulsarServiceList().get(0); + pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + + // Create a new namespace + String namespaceName = "tnx/testSegmentedSnapshotWithoutCreatingOldSnapshotTopic"; + admin.namespaces().createNamespace(namespaceName); + + // Create a new topic in the namespace + String topicName = "persistent://" + namespaceName + "/newTopic"; + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + + // Check that the __transaction_buffer_snapshot topic is not created in the same namespace + String transactionBufferSnapshotTopic = "persistent://" + namespaceName + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT; + try { + admin.topics().getStats(transactionBufferSnapshotTopic); + fail("The __transaction_buffer_snapshot topic should not exist"); + } catch (PulsarAdminException e) { + assertEquals(e.getStatusCode(), 404); + } + + // Destroy the namespace after the test + admin.namespaces().deleteNamespace(namespaceName, true); + } } From 197d35e3673a7e9fa50c20587d5824a4f3378c7a Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 17 May 2023 11:28:32 -0500 Subject: [PATCH 076/494] [feat] OIDC: support JWKS refresh for missing Key ID (#20338) ### Motivation When the `AuthenticationProviderOpenID` encounters an unknown Key ID (known as the `kid` in the JWT) for a trusted token issuer, the token is rejected with the following error `java.lang.IllegalArgumentException: No JWK found for Key ID `. This behavior is technically valid, but it isn't ideal because it is possible that the Identity Provider has issued new signing keys and started using them before the Pulsar authentication provider has refreshed its cache. This PR introduces a behavior to retrieve This PR adds a configuration called `openIDKeyIdCacheMissRefreshSeconds` that represents the length of time that must pass before the provider will reload the JWKS from the Identity Provider. The `openIDKeyIdCacheMissRefreshSeconds` setting limits the impact of an attacker invalidating the JWKS cache. When `openIDKeyIdCacheMissRefreshSeconds <= 0`, the JWKS will be refreshed for any missing key id when the issuer is trusted. This is only meant for testing. ### Modifications * Add `openIDKeyIdCacheMissRefreshSeconds` setting and default it to 5 minutes. * Add functionality to invalidate and refresh the cache when a token has an unknown `kid` for a trusted issuer. ### Verifying this change New tests are added. ### Does this pull request potentially affect one of the following parts: This adds a new configuration, but it is very minor. ### Documentation - [x] `doc-required` ### Matching PR in forked repository PR in forked repository: Skipping since tests passed already on my local machine (cherry picked from commit d92010c00e30199c5cd53021e20ba5d7bb36701c) --- .../oidc/AuthenticationProviderOpenID.java | 2 + .../broker/authentication/oidc/JwksCache.java | 48 +++++++-- ...ticationProviderOpenIDIntegrationTest.java | 100 ++++++++++++++++++ 3 files changed, 144 insertions(+), 6 deletions(-) diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java index 00ec09bd1817f..2078666a08dd9 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java @@ -133,6 +133,8 @@ public class AuthenticationProviderOpenID implements AuthenticationProvider { static final int CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT = 18 * 60 * 60; static final String CACHE_EXPIRATION_SECONDS = "openIDCacheExpirationSeconds"; static final int CACHE_EXPIRATION_SECONDS_DEFAULT = 24 * 60 * 60; + static final String KEY_ID_CACHE_MISS_REFRESH_SECONDS = "openIDKeyIdCacheMissRefreshSeconds"; + static final int KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT = 5 * 60; static final String HTTP_CONNECTION_TIMEOUT_MILLIS = "openIDHttpConnectionTimeoutMillis"; static final int HTTP_CONNECTION_TIMEOUT_MILLIS_DEFAULT = 10_000; static final String HTTP_READ_TIMEOUT_MILLIS = "openIDHttpReadTimeoutMillis"; diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java index b5e038342c2e9..73934e9c1e05e 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/JwksCache.java @@ -24,6 +24,8 @@ import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_REFRESH_AFTER_WRITE_SECONDS_DEFAULT; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.CACHE_SIZE_DEFAULT; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS; +import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT; import static org.apache.pulsar.broker.authentication.oidc.AuthenticationProviderOpenID.incrementFailureMetric; import static org.apache.pulsar.broker.authentication.oidc.ConfigUtils.getConfigValueAsInt; import com.auth0.jwk.Jwk; @@ -43,6 +45,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import javax.naming.AuthenticationException; import org.apache.pulsar.broker.ServiceConfiguration; @@ -52,7 +55,8 @@ public class JwksCache { // Map from an issuer's JWKS URI to its JWKS. When the Issuer is not empty, use the fallback client. private final AsyncLoadingCache, List> cache; - + private final ConcurrentHashMap, Long> jwksLastRefreshTime = new ConcurrentHashMap<>(); + private final long keyIdCacheMissRefreshNanos; private final ObjectReader reader = new ObjectMapper().readerFor(HashMap.class); private final AsyncHttpClient httpClient; private final OpenidApi openidApi; @@ -61,7 +65,8 @@ public class JwksCache { // Store the clients this.httpClient = httpClient; this.openidApi = apiClient != null ? new OpenidApi(apiClient) : null; - + keyIdCacheMissRefreshNanos = TimeUnit.SECONDS.toNanos(getConfigValueAsInt(config, + KEY_ID_CACHE_MISS_REFRESH_SECONDS, KEY_ID_CACHE_MISS_REFRESH_SECONDS_DEFAULT)); // Configure the cache int maxSize = getConfigValueAsInt(config, CACHE_SIZE, CACHE_SIZE_DEFAULT); int refreshAfterWriteSeconds = getConfigValueAsInt(config, CACHE_REFRESH_AFTER_WRITE_SECONDS, @@ -69,6 +74,8 @@ public class JwksCache { int expireAfterSeconds = getConfigValueAsInt(config, CACHE_EXPIRATION_SECONDS, CACHE_EXPIRATION_SECONDS_DEFAULT); AsyncCacheLoader, List> loader = (jwksUri, executor) -> { + // Store the time of the retrieval, even though it might be a little early or the call might fail. + jwksLastRefreshTime.put(jwksUri, System.nanoTime()); if (jwksUri.isPresent()) { return getJwksFromJwksUri(jwksUri.get()); } else { @@ -87,7 +94,37 @@ CompletableFuture getJwk(String jwksUri, String keyId) { incrementFailureMetric(AuthenticationExceptionCode.ERROR_RETRIEVING_PUBLIC_KEY); return CompletableFuture.failedFuture(new IllegalArgumentException("jwksUri must not be null.")); } - return cache.get(Optional.of(jwksUri)).thenApply(jwks -> getJwkForKID(jwks, keyId)); + return getJwkAndMaybeReload(Optional.of(jwksUri), keyId, false); + } + + /** + * Retrieve the JWK for the given key ID from the given JWKS URI. If the key ID is not found, and failOnMissingKeyId + * is false, then the JWK will be reloaded from the JWKS URI and the key ID will be searched for again. + */ + private CompletableFuture getJwkAndMaybeReload(Optional maybeJwksUri, + String keyId, + boolean failOnMissingKeyId) { + return cache + .get(maybeJwksUri) + .thenCompose(jwks -> { + try { + return CompletableFuture.completedFuture(getJwkForKID(maybeJwksUri, jwks, keyId)); + } catch (IllegalArgumentException e) { + if (failOnMissingKeyId) { + throw e; + } else { + Long lastRefresh = jwksLastRefreshTime.get(maybeJwksUri); + if (lastRefresh == null || System.nanoTime() - lastRefresh > keyIdCacheMissRefreshNanos) { + // In this case, the key ID was not found, but we haven't refreshed the JWKS in a while, + // so it is possible the key ID was added. Refresh the JWKS and try again. + cache.synchronous().invalidate(maybeJwksUri); + } + // There is a small race condition where the JWKS could be refreshed by another thread, + // so we retry getting the JWK, even though we might not have invalidated the cache. + return getJwkAndMaybeReload(maybeJwksUri, keyId, true); + } + } + }); } private CompletableFuture> getJwksFromJwksUri(String jwksUri) { @@ -119,8 +156,7 @@ CompletableFuture getJwkFromKubernetesApiServer(String keyId) { return CompletableFuture.failedFuture(new AuthenticationException( "Failed to retrieve public key from Kubernetes API server: Kubernetes fallback is not enabled.")); } - return cache.get(Optional.empty(), (__, executor) -> getJwksFromKubernetesApiServer()) - .thenApply(jwks -> getJwkForKID(jwks, keyId)); + return getJwkAndMaybeReload(Optional.empty(), keyId, false); } private CompletableFuture> getJwksFromKubernetesApiServer() { @@ -170,7 +206,7 @@ public void onDownloadProgress(long bytesRead, long contentLength, boolean done) return future; } - private Jwk getJwkForKID(List jwks, String keyId) { + private Jwk getJwkForKID(Optional maybeJwksUri, List jwks, String keyId) { for (Jwk jwk : jwks) { if (jwk.getId().equals(keyId)) { return jwk; diff --git a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java index 0075d70f599d3..d2d2de1a1149d 100644 --- a/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java +++ b/pulsar-broker-auth-oidc/src/test/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenIDIntegrationTest.java @@ -29,6 +29,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.impl.DefaultJwtBuilder; import io.jsonwebtoken.io.Decoders; @@ -58,6 +59,7 @@ import org.apache.pulsar.common.api.AuthData; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; /** @@ -75,6 +77,7 @@ public class AuthenticationProviderOpenIDIntegrationTest { // The valid issuer String issuer; String issuerWithTrailingSlash; + String issuerWithMissingKid; // This issuer is configured to return an issuer in the openid-configuration // that does not match the issuer on the token String issuerThatFails; @@ -90,6 +93,7 @@ void beforeClass() throws IOException { server.start(); issuer = server.baseUrl(); issuerWithTrailingSlash = issuer + "/trailing-slash/"; + issuerWithMissingKid = issuer + "/missing-kid"; issuerThatFails = issuer + "/fail"; issuerK8s = issuer + "/k8s"; @@ -183,6 +187,50 @@ void beforeClass() throws IOException { } """.formatted(validJwk, n, e, invalidJwk)))); + server.stubFor( + get(urlEqualTo("/missing-kid/.well-known/openid-configuration")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "%s", + "jwks_uri": "%s/keys" + } + """.formatted(issuerWithMissingKid, issuerWithMissingKid)))); + + // Set up JWKS endpoint where it first responds without the KID, then with the KID. This is a stateful stub. + // Note that the state machine is circular to make it easier to verify the two code paths that rely on + // this logic. + server.stubFor( + get(urlMatching( "/missing-kid/keys")) + .inScenario("Changing KIDs") + .whenScenarioStateIs(Scenario.STARTED) + .willSetStateTo("serve-kid") + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody("{\"keys\":[]}"))); + server.stubFor( + get(urlMatching( "/missing-kid/keys")) + .inScenario("Changing KIDs") + .whenScenarioStateIs("serve-kid") + .willSetStateTo(Scenario.STARTED) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody( + """ + { + "keys" : [ + { + "kid":"%s", + "kty":"RSA", + "alg":"RS256", + "n":"%s", + "e":"%s" + } + ] + } + """.formatted(validJwk, n, e)))); + ServiceConfiguration conf = new ServiceConfiguration(); conf.setAuthenticationEnabled(true); conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); @@ -207,6 +255,12 @@ void afterClass() { server.stop(); } + @BeforeMethod + public void beforeMethod() { + // Scenarios are stateful. Start each test with the correct state. + server.resetScenarios(); + } + @Test public void testTokenWithValidJWK() throws Exception { String role = "superuser"; @@ -268,6 +322,52 @@ public void testTokenWithInvalidIssuer() throws Exception { assertTrue(e.getCause() instanceof AuthenticationException, "Found exception: " + e.getCause()); } } + @Test + public void testKidCacheMissWhenRefreshConfigZero() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + // Allows us to retrieve the JWK immediately after the cache miss of the KID + props.setProperty(AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS, "0"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuerWithMissingKid); + + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + String token = generateToken(validJwk, issuerWithMissingKid, role, "allowed-audience", 0L, 0L, 10000L); + assertEquals(role, provider.authenticateAsync(new AuthenticationDataCommand(token)).get()); + } + + @Test + public void testKidCacheMissWhenRefreshConfigLongerThanDelta() throws Exception { + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setAuthenticationEnabled(true); + conf.setAuthenticationProviders(Set.of(AuthenticationProviderOpenID.class.getName())); + Properties props = conf.getProperties(); + props.setProperty(AuthenticationProviderOpenID.REQUIRE_HTTPS, "false"); + // This value is high enough that the provider will not refresh the JWK + props.setProperty(AuthenticationProviderOpenID.KEY_ID_CACHE_MISS_REFRESH_SECONDS, "100"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_AUDIENCES, "allowed-audience"); + props.setProperty(AuthenticationProviderOpenID.ALLOWED_TOKEN_ISSUERS, issuerWithMissingKid); + + AuthenticationProviderOpenID provider = new AuthenticationProviderOpenID(); + provider.initialize(conf); + + String role = "superuser"; + String token = generateToken(validJwk, issuerWithMissingKid, role, "allowed-audience", 0L, 0L, 10000L); + try { + provider.authenticateAsync(new AuthenticationDataCommand(token)).get(); + fail("Expected exception"); + } catch (ExecutionException e) { + assertTrue(e.getCause() instanceof IllegalArgumentException, "Found exception: " + e.getCause()); + assertTrue(e.getCause().getMessage().contains("No JWK found for Key ID valid"), + "Found exception: " + e.getCause()); + } + } @Test public void testKubernetesApiServerAsDiscoverTrustedIssuerSuccess() throws Exception { From b8db81a6b7bc57ea4b2ceeb6fe32fdf7da613c3a Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 10 May 2023 21:16:17 +0800 Subject: [PATCH 077/494] [fix][fn] Make pulsar-admin support update py/go with package url (#19897) (cherry picked from commit fb9c4d034bfdcfdf256720ec2490c737cae53839) --- .../main/java/org/apache/pulsar/admin/cli/CmdFunctions.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 05bab9c6f198b..d951abe0fb2a8 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -1030,6 +1030,10 @@ void runCmd() throws Exception { updateOptions.setUpdateAuthData(updateAuthData); if (Utils.isFunctionPackageUrlSupported(functionConfig.getJar())) { getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getJar(), updateOptions); + } else if (Utils.isFunctionPackageUrlSupported(functionConfig.getPy())) { + getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getPy(), updateOptions); + } else if (Utils.isFunctionPackageUrlSupported(functionConfig.getGo())) { + getAdmin().functions().updateFunctionWithUrl(functionConfig, functionConfig.getGo(), updateOptions); } else { getAdmin().functions().updateFunction(functionConfig, userCodeFile, updateOptions); } From b190a8acc4b75ed37c9696dd60a709456b3f93d8 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 25 May 2023 03:46:31 -0500 Subject: [PATCH 078/494] [fix][broker] partitioned __change_events topic is policy topic (#20392) (cherry picked from commit 9918bced4465e0b0746a7959550c90cb76ae945f) --- .../service/persistent/PersistentTopic.java | 2 +- .../common/naming/SystemTopicNames.java | 6 +-- .../common/naming/SystemTopicNamesTest.java | 47 +++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 15854f55c5cd1..98e51a2e3ed6e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -819,7 +819,7 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St } try { - if (!topic.endsWith(SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME) + if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic) && !checkSubscriptionTypesEnable(subType)) { return FutureUtil.failedFuture( new NotAllowedException("Topic[{" + topic + "}] doesn't support " diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java index 8fc7d014b5784..716d9bc31facb 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/SystemTopicNames.java @@ -81,7 +81,7 @@ public static boolean isTopicPoliciesSystemTopic(String topic) { if (topic == null) { return false; } - return TopicName.get(topic).getLocalName().equals(NAMESPACE_EVENTS_LOCAL_NAME); + return TopicName.getPartitionedTopicName(topic).getLocalName().equals(NAMESPACE_EVENTS_LOCAL_NAME); } public static boolean isTransactionInternalName(TopicName topicName) { @@ -92,7 +92,7 @@ public static boolean isTransactionInternalName(TopicName topicName) { } public static boolean isSystemTopic(TopicName topicName) { - TopicName nonePartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); - return isEventSystemTopic(nonePartitionedTopicName) || isTransactionInternalName(nonePartitionedTopicName); + TopicName nonPartitionedTopicName = TopicName.get(topicName.getPartitionedTopicName()); + return isEventSystemTopic(nonPartitionedTopicName) || isTransactionInternalName(nonPartitionedTopicName); } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java new file mode 100644 index 0000000000000..92d93021973b1 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/SystemTopicNamesTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.naming; + +import static org.testng.AssertJUnit.assertEquals; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test +public class SystemTopicNamesTest { + + @DataProvider(name = "topicPoliciesSystemTopicNames") + public static Object[][] topicPoliciesSystemTopicNames() { + return new Object[][] { + {"persistent://public/default/__change_events", true}, + {"persistent://public/default/__change_events-partition-0", true}, + {"persistent://random-tenant/random-ns/__change_events", true}, + {"persistent://random-tenant/random-ns/__change_events-partition-1", true}, + {"persistent://public/default/not_really__change_events", false}, + {"persistent://public/default/__change_events-diff-suffix", false}, + {"persistent://a/b/not_really__change_events", false}, + }; + } + + @Test(dataProvider = "topicPoliciesSystemTopicNames") + public void testIsTopicPoliciesSystemTopic(String topicName, boolean expectedResult) { + assertEquals(expectedResult, SystemTopicNames.isTopicPoliciesSystemTopic(topicName)); + assertEquals(expectedResult, SystemTopicNames.isSystemTopic(TopicName.get(topicName))); + assertEquals(expectedResult, SystemTopicNames.isEventSystemTopic(TopicName.get(topicName))); + } +} From 42d9aa59e82f921441edbf54f7b2127a7fa235c8 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 26 May 2023 18:25:52 +0300 Subject: [PATCH 079/494] [fix][ci] Update nar maven plugin version to fix excessive downloads (#20410) (cherry picked from commit 795eb51762b417ea72bf201cdfe5c7585a8c3320) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 84c8977f51968..c4140f67a20e9 100644 --- a/pom.xml +++ b/pom.xml @@ -284,7 +284,7 @@ flexible messaging model and an intuitive client API. 3.4.1 3.1.0 1.1.0 - 1.3.4 + 1.5.0 3.1.2 4.0.2 3.4.3 From b54a40a86366d3f242b92a9bf95f2c3d3edf132b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 28 Apr 2023 20:55:19 +0300 Subject: [PATCH 080/494] [improve][ci] Disable Maven http connection pooling on CI also for newer Maven versions (#20198) (cherry picked from commit 7ccd1ba119b1349de4f865b0bc0e1065c6769950) --- .github/workflows/ci-go-functions.yaml | 2 +- .github/workflows/ci-maven-cache-update.yaml | 2 +- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- .github/workflows/pulsar-ci-flaky.yaml | 2 +- .github/workflows/pulsar-ci.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index 1ff0c8eef57bf..50f003e27315f 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -33,7 +33,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: preconditions: diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml index a5a88e2e0b8a6..0bf1c4148b2c9 100644 --- a/.github/workflows/ci-maven-cache-update.yaml +++ b/.github/workflows/ci-maven-cache-update.yaml @@ -42,7 +42,7 @@ on: - cron: '30 */12 * * *' env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: update-maven-dependencies-cache: diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 8aec501f8d868..9f1ba2cc996f8 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: run-owasp-dependency-check: diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 198823c3008d1..42a51b9a00535 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -37,7 +37,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 3e2a67e3cf070..2d4792bfa15fd 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -37,7 +37,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job # it's possible to rerun individual failed jobs when the build artifacts are available # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR From 593e55f795b05b64f3e40d911ffc36d9c455b7ec Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 24 May 2023 17:29:05 +0300 Subject: [PATCH 081/494] [improve][ci] Split Pulsar IO unit test job to multiple jobs (#20384) (cherry picked from commit fd36fc1f0d9c12201e5375982ced9ee4b61af944) --- .github/workflows/pulsar-ci.yaml | 4 ++++ build/run_unit_group.sh | 12 ++++++++++++ pom.xml | 14 ++++++++++++++ pulsar-io/pom.xml | 27 +++++++++++++++++++++------ 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 2d4792bfa15fd..8cf8e64a85da4 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -202,6 +202,10 @@ jobs: - name: Pulsar IO group: PULSAR_IO timeout: 75 + - name: Pulsar IO - Elastic Search + group: PULSAR_IO_ELASTIC + - name: Pulsar IO - Kafka Connect Adaptor + group: PULSAR_IO_KAFKA_CONNECT - name: Pulsar Client group: CLIENT diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index ba49820ed1d33..69434b011b37e 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -188,6 +188,18 @@ function test_group_pulsar_io() { echo "::endgroup::" } +function test_group_pulsar_io_elastic() { + echo "::group::Running elastic-search tests" + mvn_test --install -Ppulsar-io-elastic-tests,-main + echo "::endgroup::" +} + +function test_group_pulsar_io_kafka_connect() { + echo "::group::Running Pulsar IO Kafka connect adaptor tests" + mvn_test --install -Ppulsar-io-kafka-connect-tests,-main + echo "::endgroup::" +} + function list_test_groups() { declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]' } diff --git a/pom.xml b/pom.xml index c4140f67a20e9..4f3ddec898924 100644 --- a/pom.xml +++ b/pom.xml @@ -2432,6 +2432,20 @@ flexible messaging model and an intuitive client API. + + pulsar-io-elastic-tests + + pulsar-io + + + + + pulsar-io-kafka-connect-tests + + pulsar-io + + + pulsar-sql-tests diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 0b6b37cb7eec4..0eaa6ad913634 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -85,22 +85,16 @@ batch-discovery-triggerers batch-data-generator common - docs aws twitter cassandra aerospike http - kafka rabbitmq kinesis hdfs3 jdbc data-generator - elastic-search - kafka-connect-adaptor - kafka-connect-adaptor-nar - debezium hdfs2 canal file @@ -117,6 +111,27 @@ + + pulsar-io-elastic-tests + + core + common + elastic-search + + + + + pulsar-io-kafka-connect-tests + + core + common + kafka + kafka-connect-adaptor + kafka-connect-adaptor-nar + debezium + + + core-modules From 9844383370198e463155e654f5e3483536f19487 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 27 May 2023 18:06:55 +0300 Subject: [PATCH 082/494] [improve][ci] Speed up OWASP dependency check in Pulsar CI workflow (#20412) (cherry picked from commit aa3bfcda6935dc2d9c69dce51f96ba80baeae0b7) --- .github/workflows/pulsar-ci.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 8cf8e64a85da4..8b2191e27e584 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1316,8 +1316,10 @@ jobs: cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. - - name: run "clean verify" to trigger dependency check - run: mvn -q -B -ntp verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' + - name: trigger dependency check + run: | + mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \ + -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' - name: Upload report uses: actions/upload-artifact@v3 From 423d279acfe176bbe613284deba3cd3b1a889b5c Mon Sep 17 00:00:00 2001 From: Clay Johnson Date: Sat, 29 Apr 2023 20:26:10 -0500 Subject: [PATCH 083/494] [improve][build] Capture local build scans on ge.apache.org to benefit from deep build insights (#20187) (cherry picked from commit 8c963e3ae3359fee824294c590a9c045f4800228) --- .github/actions/gradle-enterprise/action.yml | 1 - .gitignore | 1 - .mvn/{ge-extensions.xml => extensions.xml} | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) rename .mvn/{ge-extensions.xml => extensions.xml} (97%) diff --git a/.github/actions/gradle-enterprise/action.yml b/.github/actions/gradle-enterprise/action.yml index 935e76d3cd645..a11e59899c756 100644 --- a/.github/actions/gradle-enterprise/action.yml +++ b/.github/actions/gradle-enterprise/action.yml @@ -29,7 +29,6 @@ runs: - run: | if [[ -n "${{ inputs.token }}" ]]; then echo "::group::Configuring Gradle Enterprise for build" - cp .mvn/ge-extensions.xml .mvn/extensions.xml echo "GRADLE_ENTERPRISE_ACCESS_KEY=${{ inputs.token }}" >> $GITHUB_ENV echo "::endgroup::" fi diff --git a/.gitignore b/.gitignore index c584baaa0a0b8..cd00c44200059 100644 --- a/.gitignore +++ b/.gitignore @@ -97,4 +97,3 @@ test-reports/ # Gradle Enterprise .mvn/.gradle-enterprise/ -.mvn/extensions.xml diff --git a/.mvn/ge-extensions.xml b/.mvn/extensions.xml similarity index 97% rename from .mvn/ge-extensions.xml rename to .mvn/extensions.xml index 1c7a1611c1bcc..3c03b32a23775 100644 --- a/.mvn/ge-extensions.xml +++ b/.mvn/extensions.xml @@ -24,7 +24,7 @@ com.gradle gradle-enterprise-maven-extension - 1.16.3 + 1.17 com.gradle From 4f72117efb0ef0811f509798074b7875e980e023 Mon Sep 17 00:00:00 2001 From: Clay Johnson Date: Thu, 4 May 2023 06:06:13 -0500 Subject: [PATCH 084/494] [improve][ci] Replace handmade action to configure Gradle Enterprise (#20218) (cherry picked from commit 010bd508cecae501fa812be6df4b3f7d33ec4be6) --- .github/actions/gradle-enterprise/action.yml | 35 ---------- .github/workflows/ci-maven-cache-update.yaml | 6 +- .../workflows/ci-owasp-dependency-check.yaml | 7 +- .github/workflows/pulsar-ci-flaky.yaml | 6 +- .github/workflows/pulsar-ci.yaml | 68 ++++--------------- 5 files changed, 16 insertions(+), 106 deletions(-) delete mode 100644 .github/actions/gradle-enterprise/action.yml diff --git a/.github/actions/gradle-enterprise/action.yml b/.github/actions/gradle-enterprise/action.yml deleted file mode 100644 index a11e59899c756..0000000000000 --- a/.github/actions/gradle-enterprise/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: Configure Gradle Enterprise integration -description: Configure Gradle Enterprise when secret GE_ACCESS_TOKEN is available -inputs: - token: - description: 'The token for accessing Gradle Enterprise' - required: true -runs: - using: composite - steps: - - run: | - if [[ -n "${{ inputs.token }}" ]]; then - echo "::group::Configuring Gradle Enterprise for build" - echo "GRADLE_ENTERPRISE_ACCESS_KEY=${{ inputs.token }}" >> $GITHUB_ENV - echo "::endgroup::" - fi - shell: bash \ No newline at end of file diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml index 0bf1c4148b2c9..15fefaf3f1645 100644 --- a/.github/workflows/ci-maven-cache-update.yaml +++ b/.github/workflows/ci-maven-cache-update.yaml @@ -49,6 +49,7 @@ jobs: name: Update Maven dependency cache for ${{ matrix.name }} env: JOB_NAME: Update Maven dependency cache for ${{ matrix.name }} + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 45 @@ -77,11 +78,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Detect changed files if: ${{ github.event_name != 'schedule' }} id: changes diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 9f1ba2cc996f8..06edbae51adde 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -32,6 +32,7 @@ jobs: name: Check ${{ matrix.branch }} env: JOB_NAME: Check ${{ matrix.branch }} + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-20.04 timeout-minutes: 45 strategy: @@ -57,12 +58,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - if: ${{ matrix.branch == 'master' }} - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Cache local Maven repository uses: actions/cache@v3 timeout-minutes: 5 diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 42a51b9a00535..16dfa4a87aa29 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -95,6 +95,7 @@ jobs: env: JOB_NAME: Flaky tests suite COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-20.04 timeout-minutes: 100 if: ${{ needs.preconditions.outputs.docs_only != 'true' }} @@ -105,11 +106,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 8b2191e27e584..7139ea4cff910 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -96,6 +96,7 @@ jobs: name: Build and License check env: JOB_NAME: Build and License check + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-20.04 timeout-minutes: 60 if: ${{ needs.preconditions.outputs.docs_only != 'true' }} @@ -106,11 +107,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -176,6 +172,7 @@ jobs: env: JOB_NAME: CI - Unit - ${{ matrix.name }} COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}" + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-20.04 timeout-minutes: ${{ matrix.timeout || 60 }} needs: ['preconditions', 'build-and-license-check'] @@ -216,11 +213,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -396,6 +388,8 @@ jobs: timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true'}} + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout uses: actions/checkout@v3 @@ -403,11 +397,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -474,6 +463,7 @@ jobs: env: JOB_NAME: CI - Integration - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} strategy: fail-fast: false matrix: @@ -518,11 +508,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -726,11 +711,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -744,6 +724,8 @@ jobs: timeout-minutes: 60 needs: ['preconditions', 'build-and-license-check'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout uses: actions/checkout@v3 @@ -751,11 +733,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -856,6 +833,7 @@ jobs: env: JOB_NAME: CI - System - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} strategy: fail-fast: false matrix: @@ -891,11 +869,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -1086,6 +1059,7 @@ jobs: env: JOB_NAME: CI Flaky - System - ${{ matrix.name }} PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} strategy: fail-fast: false matrix: @@ -1103,11 +1077,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -1213,11 +1182,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Install gh-actions-artifact-client.js uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master @@ -1231,6 +1195,8 @@ jobs: timeout-minutes: 120 needs: ['preconditions', 'integration-tests'] if: ${{ needs.preconditions.outputs.docs_only != 'true' }} + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout uses: actions/checkout@v3 @@ -1238,11 +1204,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Cache Maven dependencies uses: actions/cache@v3 timeout-minutes: 5 @@ -1269,6 +1230,8 @@ jobs: timeout-minutes: 120 needs: [ 'preconditions', 'integration-tests' ] if: ${{ needs.preconditions.outputs.need_owasp == 'true' }} + env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout uses: actions/checkout@v3 @@ -1276,11 +1239,6 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - - name: Configure Gradle Enterprise - uses: ./.github/actions/gradle-enterprise - with: - token: ${{ secrets.GE_ACCESS_TOKEN }} - - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} From bc45b852acd0b6518856a4d7d81bc2283c4e58d5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 26 May 2023 21:21:07 +0300 Subject: [PATCH 085/494] [fix][sec] Upgrade sqlite-jdbc to resolve CVE-2023-32697 (#20411) (cherry picked from commit a953027aad38c9f54e952133949280ec2f4c04e8) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4f3ddec898924..a68681888614b 100644 --- a/pom.xml +++ b/pom.xml @@ -180,7 +180,7 @@ flexible messaging model and an intuitive client API. 2.10.10 2.5.0 5.1.0 - 3.36.0.3 + 3.42.0.0 8.0.11 42.5.1 0.3.2-patch11 From 596213615dbeb8c95cfdb09eae586b988ccd54e1 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 10 May 2023 08:06:23 +0300 Subject: [PATCH 086/494] [improve][build] Upgrade maven surefire plugin and other build/test plugins/libs including TestNG version (#20270) (cherry picked from commit 152b4a2612b50c64f5b953c06b5fdd49fa50772a) --- .mvn/extensions.xml | 2 +- buildtools/pom.xml | 17 +++++++++----- pom.xml | 23 +++++++++++-------- .../pom.xml | 7 ------ 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 3c03b32a23775..872764f899827 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -24,7 +24,7 @@ com.gradle gradle-enterprise-maven-extension - 1.17 + 1.17.1 com.gradle diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 695ae694001e6..c9e76918ddbc2 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -25,7 +25,7 @@ org.apache apache - 23 + 29 @@ -39,13 +39,12 @@ 2023-04-12T05:18:48Z 1.8 1.8 - 3.0.0-M3 + 3.1.0 2.18.0 1.7.32 - 7.7.0 + 7.7.1 3.11 4.1 - 3.4.0 8.37 3.1.2 4.1.89.Final @@ -183,11 +182,17 @@ ${test.additional.args} + + + org.apache.maven.surefire + surefire-testng + ${surefire.version} + + org.apache.maven.plugins maven-shade-plugin - ${maven-shade-plugin.version} true true @@ -255,7 +260,7 @@ org.apache.maven.wagon wagon-ssh-external - 2.10 + 3.5.3 diff --git a/pom.xml b/pom.xml index a68681888614b..1fadd76622c59 100644 --- a/pom.xml +++ b/pom.xml @@ -256,7 +256,7 @@ flexible messaging model and an intuitive client API. 3.2.13 1.1.1 - 7.7.0 + 7.7.1 3.12.4 3.25.0-GA 1.5.0 @@ -272,14 +272,13 @@ flexible messaging model and an intuitive client API. 3.0.0 4.1 1.0 - 3.1.0 + 3.3.0 - - 3.0.0-M3 - 3.4.2 - 3.10.1 - 3.4.0 + 3.1.0 + 3.5.0 + 3.11.0 + 3.5.0 2.3.0 3.4.1 3.1.0 @@ -287,7 +286,7 @@ flexible messaging model and an intuitive client API. 1.5.0 3.1.2 4.0.2 - 3.4.3 + 3.5.3 1.7.0 0.8.8 4.7.3.0 @@ -1488,7 +1487,6 @@ flexible messaging model and an intuitive client API. UTF-8 true true - true false @@ -1541,6 +1539,13 @@ flexible messaging model and an intuitive client API. + + + org.apache.maven.surefire + surefire-testng + ${surefire.version} + + diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index 73692050a8b77..1e76bce988808 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -64,13 +64,6 @@ true - - org.sonatype.plugins - nexus-staging-maven-plugin - - true - - From f1dc2cfe7d4e6c3f01db320024c92b6881df8ed8 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 30 May 2023 07:08:41 +0300 Subject: [PATCH 087/494] [improve][misc] Upgrade Netty to 4.1.93.Final (#20423) (cherry picked from commit e38091044c428af002b16110531497e2abc897d2) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 62 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 58 ++++++++--------- pom.xml | 4 +- pulsar-sql/presto-distribution/LICENSE | 60 +++++++++--------- 5 files changed, 93 insertions(+), 93 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index c9e76918ddbc2..62d805b65fa2b 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -47,7 +47,7 @@ 4.1 8.37 3.1.2 - 4.1.89.Final + 4.1.93.Final 4.2.3 31.0.1-jre 1.10.12 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index de10128ec3686..a0db421c624e1 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -289,37 +289,37 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.11.jar - org.apache.commons-commons-text-1.10.0.jar * Netty - - io.netty-netty-buffer-4.1.89.Final.jar - - io.netty-netty-codec-4.1.89.Final.jar - - io.netty-netty-codec-dns-4.1.89.Final.jar - - io.netty-netty-codec-http-4.1.89.Final.jar - - io.netty-netty-codec-http2-4.1.89.Final.jar - - io.netty-netty-codec-socks-4.1.89.Final.jar - - io.netty-netty-codec-haproxy-4.1.89.Final.jar - - io.netty-netty-common-4.1.89.Final.jar - - io.netty-netty-handler-4.1.89.Final.jar - - io.netty-netty-handler-proxy-4.1.89.Final.jar - - io.netty-netty-resolver-4.1.89.Final.jar - - io.netty-netty-resolver-dns-4.1.89.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.89.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.89.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.89.Final.jar - - io.netty-netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.89.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.89.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar - - io.netty-netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar - - io.netty-netty-tcnative-classes-2.0.56.Final.jar - - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.18.Final.jar - - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar - - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar + - io.netty-netty-buffer-4.1.93.Final.jar + - io.netty-netty-codec-4.1.93.Final.jar + - io.netty-netty-codec-dns-4.1.93.Final.jar + - io.netty-netty-codec-http-4.1.93.Final.jar + - io.netty-netty-codec-http2-4.1.93.Final.jar + - io.netty-netty-codec-socks-4.1.93.Final.jar + - io.netty-netty-codec-haproxy-4.1.93.Final.jar + - io.netty-netty-common-4.1.93.Final.jar + - io.netty-netty-handler-4.1.93.Final.jar + - io.netty-netty-handler-proxy-4.1.93.Final.jar + - io.netty-netty-resolver-4.1.93.Final.jar + - io.netty-netty-resolver-dns-4.1.93.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.93.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.93.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.93.Final.jar + - io.netty-netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar + - io.netty-netty-transport-native-epoll-4.1.93.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.93.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.61.Final.jar + - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar + - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar + - io.netty-netty-tcnative-classes-2.0.61.Final.jar + - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.21.Final.jar + - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar + - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar * Prometheus client - io.prometheus.jmx-collector-0.16.1.jar - io.prometheus-simpleclient-0.16.0.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 3021df8c63dee..3ccbd9316339f 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -348,35 +348,35 @@ The Apache Software License, Version 2.0 - commons-text-1.10.0.jar - commons-compress-1.21.jar * Netty - - netty-buffer-4.1.89.Final.jar - - netty-codec-4.1.89.Final.jar - - netty-codec-dns-4.1.89.Final.jar - - netty-codec-http-4.1.89.Final.jar - - netty-codec-socks-4.1.89.Final.jar - - netty-codec-haproxy-4.1.89.Final.jar - - netty-common-4.1.89.Final.jar - - netty-handler-4.1.89.Final.jar - - netty-handler-proxy-4.1.89.Final.jar - - netty-resolver-4.1.89.Final.jar - - netty-resolver-dns-4.1.89.Final.jar - - netty-transport-4.1.89.Final.jar - - netty-transport-classes-epoll-4.1.89.Final.jar - - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.89.Final.jar - - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final.jar - - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.56.Final.jar - - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar - - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar - - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.89.Final.jar - - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar + - netty-buffer-4.1.93.Final.jar + - netty-codec-4.1.93.Final.jar + - netty-codec-dns-4.1.93.Final.jar + - netty-codec-http-4.1.93.Final.jar + - netty-codec-socks-4.1.93.Final.jar + - netty-codec-haproxy-4.1.93.Final.jar + - netty-common-4.1.93.Final.jar + - netty-handler-4.1.93.Final.jar + - netty-handler-proxy-4.1.93.Final.jar + - netty-resolver-4.1.93.Final.jar + - netty-resolver-dns-4.1.93.Final.jar + - netty-transport-4.1.93.Final.jar + - netty-transport-classes-epoll-4.1.93.Final.jar + - netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.93.Final.jar + - netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final.jar + - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.61.Final.jar + - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar + - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar + - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.93.Final.jar + - netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/pom.xml b/pom.xml index 1fadd76622c59..44e8d0aaca6fa 100644 --- a/pom.xml +++ b/pom.xml @@ -139,8 +139,8 @@ flexible messaging model and an intuitive client API. 1.1.8.4 4.1.12.1 5.1.0 - 4.1.89.Final - 0.0.18.Final + 4.1.93.Final + 0.0.21.Final 9.4.51.v20230217 2.5.2 2.34 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d41155a321d65..e737fbf6ee2e6 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -231,37 +231,37 @@ The Apache Software License, Version 2.0 - commons-compress-1.21.jar - commons-lang3-3.11.jar * Netty - - netty-buffer-4.1.89.Final.jar - - netty-codec-4.1.89.Final.jar - - netty-codec-dns-4.1.89.Final.jar - - netty-codec-http-4.1.89.Final.jar - - netty-codec-haproxy-4.1.89.Final.jar - - netty-codec-socks-4.1.89.Final.jar - - netty-handler-proxy-4.1.89.Final.jar - - netty-common-4.1.89.Final.jar - - netty-handler-4.1.89.Final.jar + - netty-buffer-4.1.93.Final.jar + - netty-codec-4.1.93.Final.jar + - netty-codec-dns-4.1.93.Final.jar + - netty-codec-http-4.1.93.Final.jar + - netty-codec-haproxy-4.1.93.Final.jar + - netty-codec-socks-4.1.93.Final.jar + - netty-handler-proxy-4.1.93.Final.jar + - netty-common-4.1.93.Final.jar + - netty-handler-4.1.93.Final.jar - netty-reactive-streams-2.0.6.jar - - netty-resolver-4.1.89.Final.jar - - netty-resolver-dns-4.1.89.Final.jar - - netty-resolver-dns-classes-macos-4.1.89.Final.jar - - netty-resolver-dns-native-macos-4.1.89.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.89.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final.jar - - netty-tcnative-boringssl-static-2.0.56.Final-linux-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-linux-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-osx-aarch_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-osx-x86_64.jar - - netty-tcnative-boringssl-static-2.0.56.Final-windows-x86_64.jar - - netty-tcnative-classes-2.0.56.Final.jar - - netty-transport-4.1.89.Final.jar - - netty-transport-classes-epoll-4.1.89.Final.jar - - netty-transport-native-epoll-4.1.89.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.89.Final.jar - - netty-transport-native-unix-common-4.1.89.Final-linux-x86_64.jar - - netty-codec-http2-4.1.89.Final.jar - - netty-incubator-transport-classes-io_uring-0.0.18.Final.jar - - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-x86_64.jar - - netty-incubator-transport-native-io_uring-0.0.18.Final-linux-aarch_64.jar + - netty-resolver-4.1.93.Final.jar + - netty-resolver-dns-4.1.93.Final.jar + - netty-resolver-dns-classes-macos-4.1.93.Final.jar + - netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final.jar + - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar + - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar + - netty-tcnative-classes-2.0.61.Final.jar + - netty-transport-4.1.93.Final.jar + - netty-transport-classes-epoll-4.1.93.Final.jar + - netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.93.Final.jar + - netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar + - netty-codec-http2-4.1.93.Final.jar + - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar + - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar + - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar * GRPC - grpc-api-1.45.1.jar - grpc-context-1.45.1.jar From c8e45aa2ae60ba6d97b2c3088b785edc56326aab Mon Sep 17 00:00:00 2001 From: "Kim, Joo Hyuk" Date: Mon, 8 May 2023 20:57:20 +0900 Subject: [PATCH 088/494] [fix][client] Seek should be thread-safe (#20242) (cherry picked from commit bc1764f9ef71dd31e8cd61c7571e493442bc6395) --- .../pulsar/client/impl/ConsumerImpl.java | 109 ++++++++++-------- .../pulsar/client/impl/ConsumerImplTest.java | 27 ++++- 2 files changed, 86 insertions(+), 50 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 199e8a9ae71b4..6c64fe0406954 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -208,6 +208,7 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private final AtomicReference clientCnxUsedForConsumerRegistration = new AtomicReference<>(); private final List previousExceptions = new CopyOnWriteArrayList(); + static ConsumerImpl newConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, @@ -251,10 +252,12 @@ static ConsumerImpl newConsumerImpl(PulsarClientImpl client, } protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, - ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, - boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, MessageId startMessageId, - long startMessageRollbackDurationInSec, Schema schema, ConsumerInterceptors interceptors, - boolean createTopicIfDoesNotExist) { + ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, + boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, + MessageId startMessageId, + long startMessageRollbackDurationInSec, Schema schema, + ConsumerInterceptors interceptors, + boolean createTopicIfDoesNotExist) { super(client, topic, conf, conf.getReceiverQueueSize(), executorProvider, subscribeFuture, schema, interceptors); this.consumerId = client.newConsumerId(); @@ -319,21 +322,21 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat } this.connectionHandler = new ConnectionHandler(this, - new BackoffBuilder() - .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), - TimeUnit.NANOSECONDS) - .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) - .setMandatoryStop(0, TimeUnit.MILLISECONDS) - .create(), + new BackoffBuilder() + .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), + TimeUnit.NANOSECONDS) + .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) + .setMandatoryStop(0, TimeUnit.MILLISECONDS) + .create(), this); this.topicName = TopicName.get(topic); if (this.topicName.isPersistent()) { this.acknowledgmentsGroupingTracker = - new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); + new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); } else { this.acknowledgmentsGroupingTracker = - NonPersistentAcknowledgmentGroupingTracker.of(); + NonPersistentAcknowledgmentGroupingTracker.of(); } if (conf.getDeadLetterPolicy() != null) { @@ -410,16 +413,16 @@ public CompletableFuture unsubscribeAsync() { log.error("[{}][{}] Failed to unsubscribe: {}", topic, subscription, e.getCause().getMessage()); setState(State.Ready); unsubscribeFuture.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("Failed to unsubscribe the subscription %s of topic %s", - topicName.toString(), subscription))); + PulsarClientException.wrap(e.getCause(), + String.format("Failed to unsubscribe the subscription %s of topic %s", + topicName.toString(), subscription))); return null; }); } else { unsubscribeFuture.completeExceptionally( - new PulsarClientException( - String.format("The client is not connected to the broker when unsubscribing the " - + "subscription %s of the topic %s", subscription, topicName.toString()))); + new PulsarClientException( + String.format("The client is not connected to the broker when unsubscribing the " + + "subscription %s of the topic %s", subscription, topicName.toString()))); } return unsubscribeFuture; } @@ -1400,7 +1403,7 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien } private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId, - MessageIdData messageId, ClientCnx cnx) { + MessageIdData messageId, ClientCnx cnx) { // Lazy task scheduling to expire incomplete chunk message if (expireTimeOfIncompleteChunkedMessageMillis > 0 && expireChunkMessageTaskScheduled.compareAndSet(false, @@ -1445,7 +1448,7 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m increaseAvailablePermits(cnx); if (expireTimeOfIncompleteChunkedMessageMillis > 0 && System.currentTimeMillis() > (msgMetadata.getPublishTime() - + expireTimeOfIncompleteChunkedMessageMillis)) { + + expireTimeOfIncompleteChunkedMessageMillis)) { doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null); } else { trackMessage(msgId); @@ -1636,7 +1639,7 @@ protected void trackMessage(Message msg) { } protected void trackMessage(MessageId messageId) { - trackMessage(messageId, 0); + trackMessage(messageId, 0); } protected void trackMessage(MessageId messageId, int redeliveryCount) { @@ -1777,7 +1780,7 @@ private ByteBuf decryptPayloadIfNeeded(MessageIdData messageId, int redeliveryCo } private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, - ClientCnx currentCnx, boolean checkMaxMessageSize) { + ClientCnx currentCnx, boolean checkMaxMessageSize) { CompressionType compressionType = msgMetadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = msgMetadata.getUncompressedSize(); @@ -1830,7 +1833,7 @@ private void discardCorruptedMessage(MessageIdImpl messageId, ClientCnx currentC } private void discardCorruptedMessage(MessageIdData messageId, ClientCnx currentCnx, - ValidationError validationError) { + ValidationError validationError) { log.error("[{}][{}] Discarding corrupted message at {}:{}", topic, subscription, messageId.getLedgerId(), messageId.getEntryId()); discardMessage(messageId, currentCnx, validationError); @@ -2022,8 +2025,8 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) String originTopicNameStr = getOriginTopicNameStr(message); TypedMessageBuilder typedMessageBuilderNew = producerDLQ.newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())) - .value(message.getData()) - .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)); + .value(message.getData()) + .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)); if (message.hasKey()) { typedMessageBuilderNew.key(message.getKey()); } @@ -2052,7 +2055,7 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) } result.complete(false); return null; - }); + }); } }, internalPinnedExecutor).exceptionally(ex -> { log.error("Dead letter producer exception with topic: {}", deadLetterPolicy.getDeadLetterTopic(), ex); @@ -2151,9 +2154,15 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, final CompletableFuture seekFuture = new CompletableFuture<>(); ClientCnx cnx = cnx(); + if (!duringSeek.compareAndSet(false, true)) { + log.warn("[{}][{}] Attempting to seek operation that is already in progress, cancelling {}", + topic, subscription, seekBy); + seekFuture.cancel(true); + return seekFuture; + } + MessageIdAdv originSeekMessageId = seekMessageId; seekMessageId = (MessageIdAdv) seekId; - duringSeek.set(true); log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy); cnx.sendRequestWithId(seek, requestId).thenRun(() -> { @@ -2171,9 +2180,9 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, log.error("[{}][{}] Failed to reset subscription: {}", topic, subscription, e.getCause().getMessage()); seekFuture.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("Failed to seek the subscription %s of the topic %s to %s", - subscription, topicName.toString(), seekBy))); + PulsarClientException.wrap(e.getCause(), + String.format("Failed to seek the subscription %s of the topic %s to %s", + subscription, topicName.toString(), seekBy))); return null; }); return seekFuture; @@ -2185,7 +2194,7 @@ public CompletableFuture seekAsync(long timestamp) { return seekAsyncCheckState(seekBy).orElseGet(() -> { long requestId = client.newRequestId(); return seekAsyncInternal(requestId, Commands.newSeek(consumerId, requestId, timestamp), - MessageId.earliest, seekBy); + MessageId.earliest, seekBy); }); } @@ -2351,10 +2360,11 @@ public CompletableFuture> getLastMessageIdsAsync() { public CompletableFuture internalGetLastMessageIdAsync() { if (getState() == State.Closing || getState() == State.Closed) { return FutureUtil - .failedFuture(new PulsarClientException.AlreadyClosedException( - String.format("The consumer %s was already closed when the subscription %s of the topic %s " - + "getting the last message id", consumerName, subscription, topicName.toString()))); - } + .failedFuture(new PulsarClientException.AlreadyClosedException( + String.format("The consumer %s was already closed when the subscription %s of the topic %s " + + "getting the last message id", consumerName, subscription, + topicName.toString()))); + } AtomicLong opTimeoutMs = new AtomicLong(client.getConfiguration().getOperationTimeoutMs()); Backoff backoff = new BackoffBuilder() @@ -2376,11 +2386,12 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, if (isConnected() && cnx != null) { if (!Commands.peerSupportsGetLastMessageId(cnx.getRemoteEndpointProtocolVersion())) { future.completeExceptionally( - new PulsarClientException.NotSupportedException( - String.format("The command `GetLastMessageId` is not supported for the protocol version %d. " - + "The consumer is %s, topic %s, subscription %s", - cnx.getRemoteEndpointProtocolVersion(), - consumerName, topicName.toString(), subscription))); + new PulsarClientException.NotSupportedException( + String.format( + "The command `GetLastMessageId` is not supported for the protocol version %d. " + + "The consumer is %s, topic %s, subscription %s", + cnx.getRemoteEndpointProtocolVersion(), + consumerName, topicName.toString(), subscription))); return; } @@ -2399,31 +2410,31 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, } if (log.isDebugEnabled()) { log.debug("[{}][{}] Successfully getLastMessageId {}:{}", - topic, subscription, lastMessageId.getLedgerId(), lastMessageId.getEntryId()); + topic, subscription, lastMessageId.getLedgerId(), lastMessageId.getEntryId()); } MessageId lastMsgId = lastMessageId.getBatchIndex() <= 0 ? new MessageIdImpl(lastMessageId.getLedgerId(), - lastMessageId.getEntryId(), lastMessageId.getPartition()) + lastMessageId.getEntryId(), lastMessageId.getPartition()) : new BatchMessageIdImpl(lastMessageId.getLedgerId(), lastMessageId.getEntryId(), - lastMessageId.getPartition(), lastMessageId.getBatchIndex()); + lastMessageId.getPartition(), lastMessageId.getBatchIndex()); future.complete(new GetLastMessageIdResponse(lastMsgId, markDeletePosition)); }).exceptionally(e -> { log.error("[{}][{}] Failed getLastMessageId command", topic, subscription); future.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("The subscription %s of the topic %s gets the last message id was failed", - subscription, topicName.toString()))); + PulsarClientException.wrap(e.getCause(), + String.format("The subscription %s of the topic %s gets the last message id was failed", + subscription, topicName.toString()))); return null; }); } else { long nextDelay = Math.min(backoff.next(), remainingTime.get()); if (nextDelay <= 0) { future.completeExceptionally( - new PulsarClientException.TimeoutException( - String.format("The subscription %s of the topic %s could not get the last message id " - + "withing configured timeout", subscription, topicName.toString()))); + new PulsarClientException.TimeoutException( + String.format("The subscription %s of the topic %s could not get the last message id " + + "withing configured timeout", subscription, topicName.toString()))); return; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java index 29d180f5f9a16..ab11675f5434c 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -26,7 +27,9 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; - +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; +import io.netty.buffer.ByteBuf; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -259,4 +262,26 @@ public void testTopicPriorityLevel() { assertThat(consumer.getPriorityLevel()).isEqualTo(1); } + + @Test(invocationTimeOut = 1000) + public void testSeekAsyncInternal() { + // given + ClientCnx cnx = mock(ClientCnx.class); + CompletableFuture clientReq = new CompletableFuture<>(); + when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())).thenReturn(clientReq); + + consumer.setClientCnx(cnx); + consumer.setState(HandlerState.State.Ready); + + // when + CompletableFuture firstResult = consumer.seekAsync(1L); + CompletableFuture secondResult = consumer.seekAsync(1L); + + clientReq.complete(null); + + // then + assertTrue(firstResult.isDone()); + assertTrue(secondResult.isCancelled()); + verify(cnx, times(1)).sendRequestWithId(any(ByteBuf.class), anyLong()); + } } From 89da706bfcdb8a9c6ba13f2f0881820ffe4d2317 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 9 May 2023 20:47:45 +0800 Subject: [PATCH 089/494] [fix][broker] Fix `UnsupportedOperationException` when update topic properties. (#20261) (cherry picked from commit 0b4c29d091fca6606490aabdccc400280b191f17) --- .../bookkeeper/mledger/impl/MetaStoreImpl.java | 3 +-- .../pulsar/broker/admin/TopicPoliciesTest.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java index fcce1f7766cc8..1bc2d2b04bed0 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImpl.java @@ -23,7 +23,6 @@ import io.netty.buffer.CompositeByteBuf; import io.netty.buffer.Unpooled; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -182,7 +181,7 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) @Override public void operationFailed(MetaStoreException e) { if (e instanceof MetadataNotFoundException) { - result.complete(Collections.emptyMap()); + result.complete(new HashMap<>()); } else { result.completeExceptionally(e); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index a84e955cc9d81..4ff2917105246 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -133,6 +133,22 @@ public void cleanup() throws Exception { super.internalCleanup(); } + @Test + public void updatePropertiesForAutoCreatedTopicTest() throws Exception { + TopicName topicName = TopicName.get( + TopicDomain.persistent.value(), + NamespaceName.get(myNamespace), + "test-" + UUID.randomUUID() + ); + String testTopic = topicName.toString(); + @Cleanup + Producer producer = pulsarClient.newProducer().topic(testTopic).create(); + HashMap properties = new HashMap<>(); + properties.put("backlogQuotaType", "message_age"); + admin.topics().updateProperties(testTopic, properties); + admin.topics().delete(topicName.toString(), true); + } + @Test public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Exception{ TopicName topicName = TopicName.get( From 952ff16859312dfd0902d7daa5be284fe4bfc86a Mon Sep 17 00:00:00 2001 From: Benjamin Pereto Date: Wed, 10 May 2023 06:50:26 +0200 Subject: [PATCH 090/494] [fix][io] add protobuf ByteString to pulsar-io jdbc core (#20259) Signed-off-by: tison Co-authored-by: tison (cherry picked from commit 849cbf3a46cc7e85fca6500c08ab6efb8697f12e) --- pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 7 +++++++ .../org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java | 5 ++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 44e8d0aaca6fa..b522ea935b001 100644 --- a/pom.xml +++ b/pom.xml @@ -183,7 +183,7 @@ flexible messaging model and an intuitive client API. 3.42.0.0 8.0.11 42.5.1 - 0.3.2-patch11 + 0.4.6 2.7.5 0.4.4-hotfix1 3.3.5 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index cda23a9114431..4f4fa23d0cc75 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -58,6 +58,13 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml + + + com.google.protobuf + protobuf-java + provided + + \ No newline at end of file diff --git a/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java b/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java index 1c98069403c52..36c3674091932 100644 --- a/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java +++ b/pulsar-io/jdbc/core/src/main/java/org/apache/pulsar/io/jdbc/BaseJdbcAutoSchemaSink.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.ByteString; import java.sql.PreparedStatement; import java.util.ArrayList; import java.util.HashMap; @@ -185,8 +186,10 @@ private static void setColumnValue(PreparedStatement statement, int index, Objec statement.setString(index, (String) value); } else if (value instanceof Short) { statement.setShort(index, (Short) value); + } else if (value instanceof ByteString) { + statement.setBytes(index, ((ByteString) value).toByteArray()); } else { - throw new Exception("Not support value type, need to add it. " + value.getClass()); + throw new Exception("Not supported value type, need to add it. " + value.getClass()); } } From e46eda04da1c6c296c1f6cbadd6ab8c152566daa Mon Sep 17 00:00:00 2001 From: Raghavender Mittapalli <14803749+syk-coder@users.noreply.github.com> Date: Wed, 10 May 2023 18:05:17 +0530 Subject: [PATCH 091/494] [fix][broker] Fix default bundle size used while setting bookie affinity (#20250) (cherry picked from commit c32caba54a5ab2c01c092ab952343f4ac405da35) --- .../org/apache/pulsar/broker/admin/impl/NamespacesBase.java | 2 +- .../pulsar/broker/service/BrokerBookieIsolationTest.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 19f8d7c437ded..8c72d0b0286d3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -883,7 +883,7 @@ protected void internalSetBookieAffinityGroup(BookieAffinityGroupData bookieAffi policies -> new LocalPolicies(policies.bundles, bookieAffinityGroup, policies.namespaceAntiAffinityGroup)) - .orElseGet(() -> new LocalPolicies(defaultBundle(), + .orElseGet(() -> new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles()), bookieAffinityGroup, null)); log.info("[{}] Successfully updated local-policies configuration: namespace={}, map={}", clientAppId(), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java index 36d4053ae497f..951892f4ebfbc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java @@ -156,6 +156,7 @@ public void testBookieIsolation() throws Exception { config.setBrokerServicePort(Optional.of(0)); config.setAdvertisedAddress("localhost"); config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups); + config.setDefaultNumberOfNamespaceBundles(8); config.setManagedLedgerDefaultEnsembleSize(2); config.setManagedLedgerDefaultWriteQuorum(2); @@ -207,6 +208,9 @@ public void testBookieIsolation() throws Exception { .bookkeeperAffinityGroupPrimary(tenantNamespaceIsolationGroups) .build()); + //Checks the namespace bundles after setting the bookie affinity + assertEquals(admin.namespaces().getBundles(ns2).getNumBundles(), config.getDefaultNumberOfNamespaceBundles()); + try { admin.namespaces().getBookieAffinityGroup(ns1); fail("ns1 should have no bookie affinity group set"); From be47c27638c842a0370623a1c827816ce5242319 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 10 May 2023 21:15:37 +0800 Subject: [PATCH 092/494] [fix][fn] Support multiple input topics for Go runtime (#20000) (cherry picked from commit f04252b84a8b594ea371645e367c491aa8e2dd80) --- pulsar-function-go/conf/conf.go | 18 +- pulsar-function-go/conf/conf.yaml | 6 +- pulsar-function-go/pf/instance.go | 4 +- pulsar-function-go/pf/instanceConf.go | 31 ++-- .../instance/go/GoInstanceConfig.java | 5 + .../src/main/resources/findbugsExclude.xml | 10 ++ .../functions/runtime/RuntimeUtils.java | 13 +- .../functions/PulsarFunctionsTest.java | 160 +++++++++++------- .../functions/PulsarFunctionsTestBase.java | 2 + .../functions/go/PulsarFunctionsGoTest.java | 5 + .../java/PulsarFunctionsJavaTest.java | 4 +- .../python/PulsarFunctionsPythonTest.java | 8 +- 12 files changed, 179 insertions(+), 87 deletions(-) diff --git a/pulsar-function-go/conf/conf.go b/pulsar-function-go/conf/conf.go index c2ff443fcc40c..d52b886b540f9 100644 --- a/pulsar-function-go/conf/conf.go +++ b/pulsar-function-go/conf/conf.go @@ -50,8 +50,8 @@ type Conf struct { SecretsMap string `json:"secretsMap" yaml:"secretsMap"` Runtime int32 `json:"runtime" yaml:"runtime"` //Deprecated - AutoACK bool `json:"autoAck" yaml:"autoAck"` - Parallelism int32 `json:"parallelism" yaml:"parallelism"` + AutoACK bool `json:"autoAck" yaml:"autoAck"` + Parallelism int32 `json:"parallelism" yaml:"parallelism"` //source config SubscriptionType int32 `json:"subscriptionType" yaml:"subscriptionType"` TimeoutMs uint64 `json:"timeoutMs" yaml:"timeoutMs"` @@ -59,10 +59,16 @@ type Conf struct { CleanupSubscription bool `json:"cleanupSubscription" yaml:"cleanupSubscription"` SubscriptionPosition int32 `json:"subscriptionPosition" yaml:"subscriptionPosition"` //source input specs - SourceSpecTopic string `json:"sourceSpecsTopic" yaml:"sourceSpecsTopic"` - SourceSchemaType string `json:"sourceSchemaType" yaml:"sourceSchemaType"` - IsRegexPatternSubscription bool `json:"isRegexPatternSubscription" yaml:"isRegexPatternSubscription"` - ReceiverQueueSize int32 `json:"receiverQueueSize" yaml:"receiverQueueSize"` + SourceInputSpecs map[string]string `json:"sourceInputSpecs" yaml:"sourceInputSpecs"` + // for backward compatibility + // Deprecated + SourceSpecTopic string `json:"sourceSpecsTopic" yaml:"sourceSpecsTopic"` + // Deprecated + SourceSchemaType string `json:"sourceSchemaType" yaml:"sourceSchemaType"` + // Deprecated + IsRegexPatternSubscription bool `json:"isRegexPatternSubscription" yaml:"isRegexPatternSubscription"` + // Deprecated + ReceiverQueueSize int32 `json:"receiverQueueSize" yaml:"receiverQueueSize"` //sink spec config SinkSpecTopic string `json:"sinkSpecsTopic" yaml:"sinkSpecsTopic"` SinkSchemaType string `json:"sinkSchemaType" yaml:"sinkSchemaType"` diff --git a/pulsar-function-go/conf/conf.yaml b/pulsar-function-go/conf/conf.yaml index 59ac9bbd51308..098e33cda1f57 100644 --- a/pulsar-function-go/conf/conf.yaml +++ b/pulsar-function-go/conf/conf.yaml @@ -43,10 +43,8 @@ subscriptionName: "" cleanupSubscription: false subscriptionPosition: 1 # source input specs -sourceSpecsTopic: persistent://public/default/topic-01 -sourceSchemaType: "" -isRegexPatternSubscription: false -receiverQueueSize: 10 +sourceInputSpecs: + persistent://public/default/topic-01: "{\"schemaType\": \"\", \"isRegexPattern\": false, \"receiverQueueSize\": {\"value\": 10}}" # sink specs config sinkSpecsTopic: persistent://public/default/topic-02 sinkSchemaType: "" diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index a82273031ecfb..138489444d160 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -439,7 +439,6 @@ func (gi *goInstance) addLogTopicHandler() { }() if gi.context.logAppender == nil { - log.Error("the logAppender is nil, if you want to use it, please specify `--log-topic` at startup.") return } @@ -571,6 +570,9 @@ func (gi *goInstance) getMatchingMetricFunc() func(lbl *prometheus_client.LabelP func (gi *goInstance) getMatchingMetricFromRegistry(metricName string) prometheus_client.Metric { filteredMetricFamilies := gi.getFilteredMetricFamilies(metricName) + if len(filteredMetricFamilies) == 0 { + return prometheus_client.Metric{} + } metricFunc := gi.getMatchingMetricFunc() matchingMetric := getFirstMatch(filteredMetricFamilies[0].Metric, metricFunc) return *matchingMetric diff --git a/pulsar-function-go/pf/instanceConf.go b/pulsar-function-go/pf/instanceConf.go index d60beef29e8d3..9d4cabfae5a9b 100644 --- a/pulsar-function-go/pf/instanceConf.go +++ b/pulsar-function-go/pf/instanceConf.go @@ -20,6 +20,7 @@ package pf import ( + "encoding/json" "fmt" "time" @@ -44,6 +45,24 @@ type instanceConf struct { } func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { + inputSpecs := make(map[string]*pb.ConsumerSpec) + // for backward compatibility + if cfg.SourceSpecTopic != "" { + inputSpecs[cfg.SourceSpecTopic] = &pb.ConsumerSpec{ + SchemaType: cfg.SourceSchemaType, + IsRegexPattern: cfg.IsRegexPatternSubscription, + ReceiverQueueSize: &pb.ConsumerSpec_ReceiverQueueSize{ + Value: cfg.ReceiverQueueSize, + }, + } + } + for topic, value := range cfg.SourceInputSpecs { + spec := &pb.ConsumerSpec{} + if err := json.Unmarshal([]byte(value), spec); err != nil { + panic(fmt.Sprintf("Failed to unmarshal consume specs: %v", err)) + } + inputSpecs[topic] = spec + } instanceConf := &instanceConf{ instanceID: cfg.InstanceID, funcID: cfg.FuncID, @@ -66,16 +85,8 @@ func newInstanceConfWithConf(cfg *conf.Conf) *instanceConf { AutoAck: cfg.AutoACK, Parallelism: cfg.Parallelism, Source: &pb.SourceSpec{ - SubscriptionType: pb.SubscriptionType(cfg.SubscriptionType), - InputSpecs: map[string]*pb.ConsumerSpec{ - cfg.SourceSpecTopic: { - SchemaType: cfg.SourceSchemaType, - IsRegexPattern: cfg.IsRegexPatternSubscription, - ReceiverQueueSize: &pb.ConsumerSpec_ReceiverQueueSize{ - Value: cfg.ReceiverQueueSize, - }, - }, - }, + SubscriptionType: pb.SubscriptionType(cfg.SubscriptionType), + InputSpecs: inputSpecs, TimeoutMs: cfg.TimeoutMs, SubscriptionName: cfg.SubscriptionName, CleanupSubscription: cfg.CleanupSubscription, diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java index abf17bdb1656d..67fe2a41d553d 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/go/GoInstanceConfig.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.instance.go; +import java.util.Map; import lombok.Getter; import lombok.Setter; import org.apache.pulsar.functions.proto.Function; @@ -53,6 +54,10 @@ public class GoInstanceConfig { private boolean cleanupSubscription; private int subscriptionPosition = Function.SubscriptionPosition.LATEST.getNumber(); + // value is the json string of ConsumerSpec + private Map sourceInputSpecs; + + // for backward compatibility private String sourceSpecsTopic = ""; private String sourceSchemaType = ""; private boolean isRegexPatternSubscription; diff --git a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml index 027affbfeb2ec..7fe247d2ab20a 100644 --- a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml @@ -542,4 +542,14 @@ + + + + + + + + + + diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java index 53ebfcbfaf068..1a014b6f5c54a 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeUtils.java @@ -38,6 +38,7 @@ import java.net.InetAddress; import java.net.URL; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -204,9 +205,15 @@ public static List getGoInstanceCmd(InstanceConfig instanceConfig, instanceConfig.getFunctionDetails().getSource().getSubscriptionPosition().getNumber()); if (instanceConfig.getFunctionDetails().getSource().getInputSpecsMap() != null) { - for (String inputTopic : instanceConfig.getFunctionDetails().getSource().getInputSpecsMap().keySet()) { - goInstanceConfig.setSourceSpecsTopic(inputTopic); + Map sourceInputSpecs = new HashMap<>(); + for (Map.Entry entry : + instanceConfig.getFunctionDetails().getSource().getInputSpecsMap().entrySet()) { + String topic = entry.getKey(); + Function.ConsumerSpec spec = entry.getValue(); + sourceInputSpecs.put(topic, JsonFormat.printer().omittingInsignificantWhitespace().print(spec)); + goInstanceConfig.setSourceSpecsTopic(topic); } + goInstanceConfig.setSourceInputSpecs(sourceInputSpecs); } if (instanceConfig.getFunctionDetails().getSource().getTimeoutMs() != 0) { @@ -304,7 +311,7 @@ public static List getCmd(InstanceConfig instanceConfig, } if (StringUtils.isNotEmpty(functionInstanceClassPath)) { - args.add(String.format("-D%s=%s", FUNCTIONS_INSTANCE_CLASSPATH, functionInstanceClassPath)); + args.add(String.format("-D%s=%s", FUNCTIONS_INSTANCE_CLASSPATH, functionInstanceClassPath)); } else { // add complete classpath for broker/worker so that the function instance can load // the functions instance dependencies separately from user code dependencies diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index eaf66974b982a..6088628aac52b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -29,10 +29,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -126,13 +129,14 @@ protected Map produceMessagesToInputTopic(String inputTopicName, return kvs; } - protected void testFunctionLocalRun(Runtime runtime) throws Exception { + protected void testFunctionLocalRun(Runtime runtime) throws Exception { if (functionRuntimeType == FunctionRuntimeType.THREAD) { return; } - String inputTopicName = "persistent://public/default/test-function-local-run-" + runtime + "-input-" + randomName(8); + String inputTopicName = + "persistent://public/default/test-function-local-run-" + runtime + "-input-" + randomName(8); String outputTopicName = "test-function-local-run-" + runtime + "-output-" + randomName(8); final int numMessages = 10; @@ -377,7 +381,8 @@ protected void testFunctionNegAck(Runtime runtime) throws Exception { if (runtime == Runtime.PYTHON) { submitFunction( - runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, EXCEPTION_PYTHON_CLASS, schema); + runtime, inputTopicName, outputTopicName, functionName, EXCEPTION_FUNCTION_PYTHON_FILE, + EXCEPTION_PYTHON_CLASS, schema); } else { submitFunction( runtime, inputTopicName, outputTopicName, functionName, null, EXCEPTION_JAVA_CLASS, schema); @@ -550,7 +555,7 @@ protected void testPublishFunction(Runtime runtime) throws Exception { final int numMessages = 10; // submit the exclamation function - switch (runtime){ + switch (runtime) { case JAVA: submitFunction( runtime, @@ -622,7 +627,8 @@ protected void testPublishFunction(Runtime runtime) throws Exception { .create(); for (int i = 0; i < numMessages; i++) { - producer.newMessage().key(String.valueOf(i)).property("count", String.valueOf(i)).value(("message-" + i).getBytes(UTF_8)).send(); + producer.newMessage().key(String.valueOf(i)).property("count", String.valueOf(i)) + .value(("message-" + i).getBytes(UTF_8)).send(); } Set expectedMessages = new HashSet<>(); @@ -662,9 +668,10 @@ protected void testPublishFunction(Runtime runtime) throws Exception { protected void testExclamationFunction(Runtime runtime, boolean isTopicPattern, boolean pyZip, + boolean multipleInput, boolean withExtraDeps) throws Exception { - if (functionRuntimeType == FunctionRuntimeType.THREAD && runtime == Runtime.PYTHON) { - // python can only run on process mode + if (functionRuntimeType == FunctionRuntimeType.THREAD && (runtime == Runtime.PYTHON || runtime == Runtime.GO)) { + // python&go can only run on process mode return; } @@ -683,22 +690,9 @@ protected void testExclamationFunction(Runtime runtime, admin.topics().createNonPartitionedTopic(outputTopicName); } if (isTopicPattern) { - @Cleanup PulsarClient client = PulsarClient.builder() - .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) - .build(); - - @Cleanup Consumer consumer1 = client.newConsumer(schema) - .topic(inputTopicName + "1") - .subscriptionType(SubscriptionType.Exclusive) - .subscriptionName("test-sub") - .subscribe(); - - @Cleanup Consumer consumer2 = client.newConsumer(schema) - .topic(inputTopicName + "2") - .subscriptionType(SubscriptionType.Exclusive) - .subscriptionName("test-sub") - .subscribe(); inputTopicName = inputTopicName + ".*"; + } else if (multipleInput) { + inputTopicName = inputTopicName + "1," + inputTopicName + "2"; } String functionName = "test-exclamation-fn-" + randomName(8); final int numMessages = 10; @@ -725,8 +719,11 @@ protected void testExclamationFunction(Runtime runtime, // get function status getFunctionStatus(functionName, numMessages, true); - // get function stats - getFunctionStats(functionName, numMessages); + if (Runtime.GO != runtime) { + // TODO: Go runtime doesn't collect `process_latency_ms_1min` metric + // get function stats + getFunctionStats(functionName, numMessages); + } // update parallelism updateFunctionParallelism(functionName, 2); @@ -787,6 +784,12 @@ private void submitFunction(Runtime runtime, } else { file = EXCLAMATION_PYTHON_FILE; } + } else if (Runtime.GO == runtime) { + if (isPublishFunction) { + file = PUBLISH_FUNCTION_GO_FILE; + } else { + file = EXCLAMATION_GO_FILE; + } } submitFunction(runtime, inputTopicName, outputTopicName, functionName, file, functionClass, inputTopicSchema); @@ -819,7 +822,7 @@ private void submitFunction(Runtime runtime, if (StringUtils.isNotEmpty(inputTopicName)) { ensureSubscriptionCreated( - inputTopicName, String.format("public/default/%s", functionName), inputTopicSchema); + inputTopicName, String.format("public/default/%s", functionName), inputTopicSchema); } CommandGenerator generator; @@ -853,7 +856,7 @@ private void submitFunction(Runtime runtime, } String command = ""; - switch (runtime){ + switch (runtime) { case JAVA: command = generator.generateCreateFunctionCommand(); break; @@ -945,8 +948,17 @@ private void ensureSubscriptionCreated(String inputTopicName, try (PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarCluster.getPlainTextServiceUrl()) .build()) { + List topics = new ArrayList<>(); + if (inputTopicName.endsWith(".*")) { + topics.add(inputTopicName.substring(0, inputTopicName.length() - 2) + "1"); + topics.add(inputTopicName.substring(0, inputTopicName.length() - 2) + "2"); + } else if (inputTopicName.contains(",")) { + topics.addAll(Arrays.asList(inputTopicName.split(","))); + } else { + topics.add(inputTopicName); + } try (Consumer ignored = client.newConsumer(inputTopicSchema) - .topic(inputTopicName) + .topic(topics.toArray(new String[0])) .subscriptionType(SubscriptionType.Shared) .subscriptionName(subscriptionName) .subscribe()) { @@ -1042,7 +1054,8 @@ private void getFunctionStats(String functionName, int numMessages) throws Excep assertEquals(functionStats.instances.get(0).getMetrics().getUserExceptionsTotal(), 0); assertTrue(functionStats.instances.get(0).getMetrics().getAvgProcessLatency() > 0); assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getReceivedTotal(), numMessages); - assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), numMessages); + assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getProcessedSuccessfullyTotal(), + numMessages); assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getSystemExceptionsTotal(), 0); assertEquals(functionStats.instances.get(0).getMetrics().getOneMin().getUserExceptionsTotal(), 0); assertTrue(functionStats.instances.get(0).getMetrics().getOneMin().getAvgProcessLatency() > 0); @@ -1071,19 +1084,29 @@ private void getFunctionInfoNotFound(String functionName) throws Exception { } private void checkSubscriptionsCleanup(String topic) throws Exception { - try { - ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd( - PulsarCluster.ADMIN_SCRIPT, - "topics", - "stats", - topic); - TopicStats topicStats = ObjectMapperFactory.getMapper().reader() - .readValue(result.getStdout(), TopicStats.class); - assertEquals(topicStats.getSubscriptions().size(), 0); - - } catch (ContainerExecException e) { - fail("Command should have exited with non-zero"); + List topics = new ArrayList<>(); + if (topic.endsWith(".*")) { + topics.add(topic.substring(0, topic.length() - 2) + "1"); + topics.add(topic.substring(0, topic.length() - 2) + "2"); + } else if (topic.contains(",")) { + topics.addAll(Arrays.asList(topic.split(","))); + } else { + topics.add(topic); } + topics.stream().forEach(t -> { + try { + ContainerExecResult result = pulsarCluster.getAnyBroker().execCmd( + PulsarCluster.ADMIN_SCRIPT, + "topics", + "stats", + t); + TopicStats topicStats = ObjectMapperFactory.getMapper().reader() + .readValue(result.getStdout(), TopicStats.class); + assertEquals(topicStats.getSubscriptions().size(), 0); + } catch (Exception e) { + fail("Command should have exited with non-zero"); + } + }); } private void checkPublisherCleanup(String topic) throws Exception { @@ -1145,7 +1168,8 @@ private void doGetFunctionStatus(String functionName, int numMessages, boolean c lastInvocationTimeGreaterThanZero = lastInvocationTimeGreaterThanZero || functionStatus.getInstances().get(i).getStatus().getLastInvocationTime() > 0; totalMessagesProcessed += functionStatus.getInstances().get(i).getStatus().getNumReceived(); - totalMessagesSuccessfullyProcessed += functionStatus.getInstances().get(i).getStatus().getNumSuccessfullyProcessed(); + totalMessagesSuccessfullyProcessed += + functionStatus.getInstances().get(i).getStatus().getNumSuccessfullyProcessed(); if (checkRestarts) { assertEquals(functionStatus.getInstances().get(i).getStatus().getNumRestarts(), 0); } @@ -1238,6 +1262,23 @@ private void publishAndConsumeMessagesBytes(String inputTopic, producer1.send(("message-" + i).getBytes(UTF_8)); } + for (int i = numMessages / 2; i < numMessages; i++) { + producer2.send(("message-" + i).getBytes(UTF_8)); + } + } else if (inputTopic.contains(",")) { + String[] topics = inputTopic.split(","); + @Cleanup Producer producer1 = client.newProducer(Schema.BYTES) + .topic(topics[0]) + .create(); + + @Cleanup Producer producer2 = client.newProducer(Schema.BYTES) + .topic(topics[1]) + .create(); + + for (int i = 0; i < numMessages / 2; i++) { + producer1.send(("message-" + i).getBytes(UTF_8)); + } + for (int i = numMessages / 2; i < numMessages; i++) { producer2.send(("message-" + i).getBytes(UTF_8)); } @@ -1420,7 +1461,8 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception { AVRO_SCHEMA_FUNCTION_PYTHON_FILE, AVRO_SCHEMA_PYTHON_CLASS, Schema.AVRO(AvroTestObject.class), - null, objectMapper.writeValueAsString(inputSpecs), "avro", null, "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject"); + null, objectMapper.writeValueAsString(inputSpecs), "avro", null, + "avro_schema_test_function.AvroTestObject", "avro_schema_test_function.AvroTestObject"); } log.info("pulsar submitFunction"); @@ -1430,7 +1472,7 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception { Set expectedSet = new HashSet<>(); log.info("test-avro-schema producer connected: " + producer.isConnected()); - for (int i = 0 ; i < numMessages ; i++) { + for (int i = 0; i < numMessages; i++) { AvroTestObject inputObject = new AvroTestObject(); inputObject.setBaseValue(i); MessageId messageId = producer.send(inputObject); @@ -1457,7 +1499,7 @@ protected void testAvroSchemaFunction(Runtime runtime) throws Exception { }); log.info("test-avro-schema consumer connected: " + consumer.isConnected()); - for (int i = 0 ; i < numMessages ; i++) { + for (int i = 0; i < numMessages; i++) { log.info("test-avro-schema consumer receive [{}] start", i); Message message = consumer.receive(); log.info("test-avro-schema consumer receive [{}] over", i); @@ -1495,7 +1537,8 @@ protected void testInitFunction(Runtime runtime) throws Exception { final int numMessages = 10; // submit the exclamation function - submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, InitializableFunction.class.getName(), schema, + submitFunction(runtime, inputTopicName, outputTopicName, functionName, null, + InitializableFunction.class.getName(), schema, Collections.singletonMap("publish-topic", outputTopicName), null, null, null, null, null); // publish and consume result @@ -1650,7 +1693,8 @@ private void publishAndConsumeMessages(String inputTopic, } - protected void testGenericObjectFunction(String function, boolean removeAgeField, boolean keyValue) throws Exception { + protected void testGenericObjectFunction(String function, boolean removeAgeField, boolean keyValue) + throws Exception { log.info("start {} function test ...", function); String ns = "public/ns-genericobject-" + randomName(8); @@ -1721,13 +1765,14 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField GenericRecord genericRecord = message.getValue(); if (keyValue) { @SuppressWarnings("unchecked") - KeyValue keyValueObject = (KeyValue) genericRecord.getNativeObject(); + KeyValue keyValueObject = + (KeyValue) genericRecord.getNativeObject(); GenericRecord key = keyValueObject.getKey(); GenericRecord value = keyValueObject.getValue(); - key.getFields().forEach(f-> { + key.getFields().forEach(f -> { log.info("key field {} value {}", f.getName(), key.getField(f.getName())); }); - value.getFields().forEach(f-> { + value.getFields().forEach(f -> { log.info("value field {} value {}", f.getName(), value.getField(f.getName())); }); assertEquals(i, key.getField("age")); @@ -1743,7 +1788,7 @@ protected void testGenericObjectFunction(String function, boolean removeAgeField } else { GenericRecord value = genericRecord; log.info("received value {}", value); - value.getFields().forEach(f-> { + value.getFields().forEach(f -> { log.info("value field {} value {}", f.getName(), value.getField(f.getName())); }); @@ -1939,13 +1984,14 @@ private void prepareDataForMergeFunction(String ns, } private void generateDataByDifferentSchema(String ns, - String baseTopic, - PulsarClient pulsarClient, - Schema schema, - T data, - int messageCnt, - ObjectNode inputSpecNode, - Map topicMsgCntMap) throws PulsarClientException { + String baseTopic, + PulsarClient pulsarClient, + Schema schema, + T data, + int messageCnt, + ObjectNode inputSpecNode, + Map topicMsgCntMap) + throws PulsarClientException { String topic = ns + "/" + baseTopic; Producer producer = pulsarClient.newProducer(schema) .topic(topic) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 033e590d6dd4e..62aa36da1b6be 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -151,6 +151,8 @@ protected static String getExclamationClass(Runtime runtime, } else { return EXCLAMATION_PYTHON_CLASS; } + } else if (Runtime.GO == runtime) { + return null; } else { throw new IllegalArgumentException("Unsupported runtime : " + runtime); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java index 9ef2398c55c00..ad8dc475aac20 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/go/PulsarFunctionsGoTest.java @@ -34,4 +34,9 @@ public void testGoFunctionLocalRun() throws Exception { testFunctionLocalRun(Runtime.GO); } + @Test(groups = {"go_function", "function"}) + public void testGoExclamationMultiInputsFunction() throws Exception { + testExclamationFunction(Runtime.GO, false, false, true, false); + } + } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java index e6f3c67ebcdb7..097a452937d27 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java @@ -98,12 +98,12 @@ private void testCustomSerdeFunction() throws Exception { @Test(groups = {"java_function", "function"}) public void testJavaExclamationFunction() throws Exception { - testExclamationFunction(Runtime.JAVA, false, false, false); + testExclamationFunction(Runtime.JAVA, false, false, false, false); } @Test(groups = {"java_function", "function"}) public void testJavaExclamationTopicPatternFunction() throws Exception { - testExclamationFunction(Runtime.JAVA, true, false, false); + testExclamationFunction(Runtime.JAVA, true, false, false, false); } @Test(groups = {"java_function", "function"}) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java index 1c75d70498609..87a52d27e8927 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/python/PulsarFunctionsPythonTest.java @@ -46,22 +46,22 @@ public void testPythonPublishFunction() throws Exception { @Test(groups = {"python_function", "function"}) public void testPythonExclamationFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, false, false); + testExclamationFunction(Runtime.PYTHON, false, false, false, false); } @Test(groups = {"python_function", "function"}) public void testPythonExclamationFunctionWithExtraDeps() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, false, true); + testExclamationFunction(Runtime.PYTHON, false, false, false, true); } @Test(groups = {"python_function", "function"}) public void testPythonExclamationZipFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, false, true, false); + testExclamationFunction(Runtime.PYTHON, false, true, false, false); } @Test(groups = {"python_function", "function"}) public void testPythonExclamationTopicPatternFunction() throws Exception { - testExclamationFunction(Runtime.PYTHON, true, false, false); + testExclamationFunction(Runtime.PYTHON, true, false, false, false); } @Test(groups = {"python_function", "function"}) From 28ddbf1780864137546e10aa3e0fec6d86e8a739 Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Thu, 11 May 2023 22:28:40 +0800 Subject: [PATCH 093/494] [fix][broker] Fix NPE cause by topic publish rate limiter. (#20302) Co-authored-by: druidliu (cherry picked from commit 4b24c9e85ad534c2087063e7873eded4f9ab7a21) --- .../java/org/apache/pulsar/broker/service/AbstractTopic.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 6245ce19eebc6..8269dc0c3d16f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -115,7 +115,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener Date: Thu, 11 May 2023 17:29:22 -0500 Subject: [PATCH 094/494] [fix][fn] Correct TLS cert config translation from broker to fn worker (#20297) ### Motivation The `initializeWorkerConfigFromBrokerConfig` method converts a `ServiceConfiguration` object to a `WorkerConfig`. This is used when the function worker is running within the broker and when pulsar is running in standalone mode. The TLS certificates that are trusted by the broker should also be trusted by the function worker, and the TLS certificates that are trusted by the broker client should be trusted by the function worker's client. ### Modifications * Improve the `PulsarFunctionTlsTest` test by adding awaitility. Before this change, the test was flaky on my machine. * Fix the mapping of broker config to function worker config. ### Verifying this change A test is added. Note that the old test doesn't technically fail due to the misconfiguration at the moment. The error is in the logs. Here is one of the stack traces that is gone after this change: ``` 2023-05-10T23:19:27,087 - WARN - [pulsar-client-io-253-3:ClientCnx@344] - [localhost/127.0.0.1:59715] Got exception io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: General OpenSslEngine problem at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:499) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:833) Caused by: javax.net.ssl.SSLHandshakeException: General OpenSslEngine problem at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.handshakeException(ReferenceCountedOpenSslEngine.java:1907) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.wrap(ReferenceCountedOpenSslEngine.java:834) at java.base/javax.net.ssl.SSLEngine.wrap(SSLEngine.java:564) at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:1042) at io.netty.handler.ssl.SslHandler.wrapNonAppData(SslHandler.java:928) at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1418) at io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1256) at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1296) at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ... 17 more Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at java.base/sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:439) at java.base/sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:306) at java.base/sun.security.validator.Validator.validate(Validator.java:264) at java.base/sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:285) at java.base/sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:144) at io.netty.handler.ssl.ReferenceCountedOpenSslClientContext$ExtendedTrustManagerVerifyCallback.verify(ReferenceCountedOpenSslClientContext.java:234) at io.netty.handler.ssl.ReferenceCountedOpenSslContext$AbstractCertificateVerifier.verify(ReferenceCountedOpenSslContext.java:779) at io.netty.internal.tcnative.CertificateVerifierTask.runTask(CertificateVerifierTask.java:36) at io.netty.internal.tcnative.SSLTask.run(SSLTask.java:48) at io.netty.internal.tcnative.SSLTask.run(SSLTask.java:42) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.runAndResetNeedTask(ReferenceCountedOpenSslEngine.java:1496) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.access$700(ReferenceCountedOpenSslEngine.java:94) at io.netty.handler.ssl.ReferenceCountedOpenSslEngine$TaskDecorator.run(ReferenceCountedOpenSslEngine.java:1471) at io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1558) at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1404) ... 21 more Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at java.base/sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141) at java.base/sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126) at java.base/java.security.cert.CertPathBuilder.build(CertPathBuilder.java:297) at java.base/sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:434) ... 35 more ``` ### Does this pull request potentially affect one of the following parts: This affects the configuration, but I think it should be classified as fixing a bug, so it is not a breaking change. ### Documentation - [x] `doc-not-needed` ### Matching PR in forked repository PR in forked repository: the modified test passes locally, so skipping forked test. (cherry picked from commit 525dd2f3b541d8bb8fe166b28b79597aae45ef4f) --- .../apache/pulsar/broker/PulsarService.java | 3 ++- .../worker/PulsarFunctionTlsTest.java | 23 +++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 5fc9920d0f215..f833674ceeb0d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1859,7 +1859,8 @@ public static WorkerConfig initializeWorkerConfigFromBrokerConfig(ServiceConfigu workerConfig.setTlsAllowInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()); workerConfig.setTlsEnabled(brokerConfig.isTlsEnabled()); workerConfig.setTlsEnableHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()); - workerConfig.setBrokerClientTrustCertsFilePath(brokerConfig.getTlsTrustCertsFilePath()); + workerConfig.setBrokerClientTrustCertsFilePath(brokerConfig.getBrokerClientTrustCertsFilePath()); + workerConfig.setTlsTrustCertsFilePath(brokerConfig.getTlsTrustCertsFilePath()); // client in worker will use this config to authenticate with broker workerConfig.setBrokerClientAuthenticationPlugin(brokerConfig.getBrokerClientAuthenticationPlugin()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index 00db6a65b16ea..246d980d6178c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -51,6 +51,7 @@ import org.apache.pulsar.functions.sink.PulsarSink; import org.apache.pulsar.functions.worker.service.WorkerServiceLoader; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -65,6 +66,7 @@ public class PulsarFunctionTlsTest { private static final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/authentication/tls/broker-key.pem"; private static final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/authentication/tls/client-cert.pem"; private static final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/authentication/tls/client-key.pem"; + private static final String CA_CERT_FILE_PATH = "./src/test/resources/authentication/tls/cacert.pem"; LocalBookkeeperEnsemble bkEnsemble; protected PulsarAdmin[] pulsarAdmins = new PulsarAdmin[BROKER_COUNT]; @@ -111,9 +113,9 @@ void setup() throws Exception { config.setAuthenticationProviders(providers); config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); - config.setTlsAllowInsecureConnection(true); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); config.setBrokerClientTlsEnabled(true); - config.setBrokerClientTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH); + config.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); config.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + ",tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); @@ -141,7 +143,6 @@ void setup() throws Exception { workerConfig.setUseTls(true); workerConfig.setTlsEnableHostnameVerification(true); workerConfig.setTlsAllowInsecureConnection(false); - workerConfig.setBrokerClientTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH); fnWorkerServices[i] = WorkerServiceLoader.load(workerConfig); configurations[i] = config; @@ -163,8 +164,7 @@ void setup() throws Exception { pulsarAdmins[i] = PulsarAdmin.builder() .serviceHttpUrl(pulsarServices[i].getWebServiceAddressTls()) - .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH) - .allowTlsInsecureConnection(true) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) .authentication(authTls) .build(); } @@ -216,6 +216,13 @@ void tearDown() throws Exception { } } + @Test + public void testTLSTrustCertsConfigMapping() throws Exception { + WorkerConfig workerConfig = fnWorkerServices[0].getWorkerConfig(); + assertEquals(workerConfig.getTlsTrustCertsFilePath(), CA_CERT_FILE_PATH); + assertEquals(workerConfig.getBrokerClientTrustCertsFilePath(), CA_CERT_FILE_PATH); + } + @Test public void testFunctionsCreation() throws Exception { @@ -233,6 +240,12 @@ public void testFunctionsCreation() throws Exception { functionConfig, jarFilePathUrl ); + // Function creation is not strongly consistent, so this test can fail with a get that is too eager and + // does not have retries. + final PulsarAdmin admin = pulsarAdmins[i]; + Awaitility.await().ignoreExceptions() + .untilAsserted(() -> admin.functions().getFunction(testTenant, "my-ns", functionName)); + FunctionConfig config = pulsarAdmins[i].functions().getFunction(testTenant, "my-ns", functionName); assertEquals(config.getTenant(), testTenant); assertEquals(config.getNamespace(), "my-ns"); From c1a854299b9d3335f815122c31ae7bf87f4efa9e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 12 May 2023 12:27:49 +0800 Subject: [PATCH 095/494] [fix][build] Fix publish image script (#20305) Signed-off-by: Zixuan Liu (cherry picked from commit 94c7bf3e57f6fc4323b36142fba651c4ad21d301) --- docker/publish.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/publish.sh b/docker/publish.sh index af0d72d4b3437..45b338d85f8ef 100755 --- a/docker/publish.sh +++ b/docker/publish.sh @@ -62,11 +62,11 @@ set -x # Fail if any of the subsequent commands fail set -e -docker tag pulsar:latest ${docker_registry_org}/pulsar:latest -docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:latest +docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:latest +docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:latest -docker tag pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION -docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION +docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION +docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION # Push all images and tags docker push ${docker_registry_org}/pulsar:latest From a08c8ee5ccbba279224e43c7c7afa8c4a00031bb Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 12 May 2023 19:29:42 +0800 Subject: [PATCH 096/494] [fix][broker] Allow Access to System Topic Metadata for Reader Creation Post-Namespace Deletion (#20304) ## Motivation After initiating the snapshot segment function, deletion of topics necessitates the activation of readers. Furthermore, these readers should be opened and deleted as they are used, which implies that we should not pre-store readers. However, after initiating the deletion of namespaces currently, it is not allowed to obtain the metadata of partition topics or lookup, making it impossible to create readers. This results in the inability to delete namespaces. ## Modification Allow the acquisition of system topic metadata after initiating namespace deletion, thus creating readers to clean up topic data. (cherry picked from commit 5d5ec947249df50bf35a78b6a2a0d3b00d97ca66) --- .../admin/impl/PersistentTopicsBase.java | 14 +++++- .../pulsar/broker/lookup/TopicLookupBase.java | 44 +++++++++++-------- .../broker/transaction/TransactionTest.java | 21 +++++++++ 3 files changed, 58 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index fcade8270cb12..de75c5f2132c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin.impl; +import static org.apache.pulsar.common.naming.SystemTopicNames.isSystemTopic; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionCoordinatorAssign; import static org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName; import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX; @@ -4408,8 +4409,13 @@ public CompletableFuture getPartitionedTopicMetadata( // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can // serve/redirect request else fail partitioned-metadata-request so, client fails while creating // producer/consumer + // It is necessary for system topic operations because system topics are used to store metadata + // and other vital information. Even after namespace starting deletion,, + // we need to access the metadata of system topics to create readers and clean up topic data. + // If we don't do this, it can prevent namespace deletion due to inaccessible readers. authorizationFuture.thenCompose(__ -> - checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject())) + checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject(), + SystemTopicNames.isSystemTopic(topicName))) .thenCompose(res -> pulsar.getBrokerService().fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName)) .thenAccept(metadata -> { @@ -4436,7 +4442,11 @@ public static CompletableFuture unsafeGetPartitionedTo // validates global-namespace contains local/peer cluster: if peer/local cluster present then lookup can // serve/redirect request else fail partitioned-metadata-request so, client fails while creating // producer/consumer - checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject()) + // It is necessary for system topic operations because system topics are used to store metadata + // and other vital information. Even after namespace starting deletion,, + // we need to access the metadata of system topics to create readers and clean up topic data. + // If we don't do this, it can prevent namespace deletion due to inaccessible readers. + checkLocalOrGetPeerReplicationCluster(pulsar, topicName.getNamespaceObject(), isSystemTopic(topicName)) .thenCompose(res -> pulsar.getBrokerService() .fetchPartitionedTopicMetadataCheckAllowAutoCreationAsync(topicName)) .thenAccept(metadata -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index 3b64d2a9f8393..bd70201cba55d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -41,6 +41,7 @@ import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.lookup.data.LookupData; import org.apache.pulsar.common.naming.NamespaceBundle; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.NamespaceOperation; @@ -221,26 +222,31 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe // (2) authorize client checkAuthorizationAsync(pulsarService, topicName, clientAppId, authenticationData).thenRun(() -> { // (3) validate global namespace + // It is necessary for system topic operations because system topics are used to store metadata + // and other vital information. Even after namespace starting deletion, + // we need to access the metadata of system topics to create readers and clean up topic data. + // If we don't do this, it can prevent namespace deletion due to inaccessible readers. checkLocalOrGetPeerReplicationCluster(pulsarService, - topicName.getNamespaceObject()).thenAccept(peerClusterData -> { - if (peerClusterData == null) { - // (4) all validation passed: initiate lookup - validationFuture.complete(null); - return; - } - // if peer-cluster-data is present it means namespace is owned by that peer-cluster and - // request should be redirect to the peer-cluster - if (StringUtils.isBlank(peerClusterData.getBrokerServiceUrl()) - && StringUtils.isBlank(peerClusterData.getBrokerServiceUrlTls())) { - validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError, - "Redirected cluster's brokerService url is not configured", - requestId)); - return; - } - validationFuture.complete(newLookupResponse(peerClusterData.getBrokerServiceUrl(), - peerClusterData.getBrokerServiceUrlTls(), true, LookupType.Redirect, - requestId, - false)); + topicName.getNamespaceObject(), SystemTopicNames.isSystemTopic(topicName)) + .thenAccept(peerClusterData -> { + if (peerClusterData == null) { + // (4) all validation passed: initiate lookup + validationFuture.complete(null); + return; + } + // if peer-cluster-data is present it means namespace is owned by that peer-cluster + // and request should be redirect to the peer-cluster + if (StringUtils.isBlank(peerClusterData.getBrokerServiceUrl()) + && StringUtils.isBlank(peerClusterData.getBrokerServiceUrlTls())) { + validationFuture.complete(newLookupErrorResponse(ServerError.MetadataError, + "Redirected cluster's brokerService url is not configured", + requestId)); + return; + } + validationFuture.complete(newLookupResponse(peerClusterData.getBrokerServiceUrl(), + peerClusterData.getBrokerServiceUrlTls(), true, + LookupType.Redirect, requestId, + false)); }).exceptionally(ex -> { Throwable throwable = FutureUtil.unwrapCompletionException(ex); if (throwable instanceof RestException restException){ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index c3533e70cf8be..c4ec2ec766e32 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -272,6 +272,27 @@ public void testCreateTransactionSystemTopic() throws Exception { } } + @Test + public void testCanDeleteNamespaceWhenEnableTxnSegmentedSnapshot() throws Exception { + // Enable the segmented snapshot feature + pulsarServiceList.get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + pulsarServiceList.get(0).getConfig().setForceDeleteNamespaceAllowed(true); + + // Create a new namespace + String namespaceName = TENANT + "/testSegmentedSnapshotWithoutCreatingOldSnapshotTopic"; + admin.namespaces().createNamespace(namespaceName); + + // Create a new topic in the namespace + String topicName = "persistent://" + namespaceName + "/newTopic"; + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).create(); + producer.close(); + + // Destroy the namespace after the test + admin.namespaces().deleteNamespace(namespaceName, true); + pulsarServiceList.get(0).getConfig().setTransactionBufferSegmentedSnapshotEnabled(false); + } + @Test public void brokerNotInitTxnManagedLedgerTopic() throws Exception { String subName = "test"; From be929f70ae4da6c61a0f613e17eb6fa68930f9c8 Mon Sep 17 00:00:00 2001 From: "Kim, Joo Hyuk" Date: Mon, 15 May 2023 16:29:50 +0900 Subject: [PATCH 097/494] [fix][broker] Fix class name typo `PrecisPublishLimiter` to "Precise" (#20310) (cherry picked from commit 7e54e9b0463f5c69846dc44892d88281d5508466) --- .../pulsar/broker/service/AbstractTopic.java | 2 +- ...imiter.java => PrecisePublishLimiter.java} | 10 +++--- .../PrecisTopicPublishRateThrottleTest.java | 2 +- ...st.java => PrecisePublishLimiterTest.java} | 30 ++++++++-------- .../service/PublishRateLimiterTest.java | 34 ++++++++++--------- 5 files changed, 40 insertions(+), 38 deletions(-) rename pulsar-broker/src/main/java/org/apache/pulsar/broker/service/{PrecisPublishLimiter.java => PrecisePublishLimiter.java} (93%) rename pulsar-broker/src/test/java/org/apache/pulsar/broker/service/{PrecisPublishLimiterTest.java => PrecisePublishLimiterTest.java} (57%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 8269dc0c3d16f..4614b846c8eee 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -1274,7 +1274,7 @@ public void updatePublishDispatcher() { || this.topicPublishRateLimiter == PublishRateLimiter.DISABLED_RATE_LIMITER) { // create new rateLimiter if rate-limiter is disabled if (preciseTopicPublishRateLimitingEnable) { - this.topicPublishRateLimiter = new PrecisPublishLimiter(publishRate, + this.topicPublishRateLimiter = new PrecisePublishLimiter(publishRate, () -> this.enableCnxAutoRead(), brokerService.pulsar().getExecutor()); } else { this.topicPublishRateLimiter = new PublishRateLimiterImpl(publishRate); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java similarity index 93% rename from pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java rename to pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java index 6e215b3b49515..ce14f6d7dd71e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisPublishLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PrecisePublishLimiter.java @@ -24,7 +24,7 @@ import org.apache.pulsar.common.util.RateLimitFunction; import org.apache.pulsar.common.util.RateLimiter; -public class PrecisPublishLimiter implements PublishRateLimiter { +public class PrecisePublishLimiter implements PublishRateLimiter { protected volatile int publishMaxMessageRate = 0; protected volatile long publishMaxByteRate = 0; // precise mode for publish rate limiter @@ -33,18 +33,18 @@ public class PrecisPublishLimiter implements PublishRateLimiter { private final RateLimitFunction rateLimitFunction; private final ScheduledExecutorService scheduledExecutorService; - public PrecisPublishLimiter(Policies policies, String clusterName, RateLimitFunction rateLimitFunction) { + public PrecisePublishLimiter(Policies policies, String clusterName, RateLimitFunction rateLimitFunction) { this.rateLimitFunction = rateLimitFunction; update(policies, clusterName); this.scheduledExecutorService = null; } - public PrecisPublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction) { + public PrecisePublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction) { this(publishRate, rateLimitFunction, null); } - public PrecisPublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction, - ScheduledExecutorService scheduledExecutorService) { + public PrecisePublishLimiter(PublishRate publishRate, RateLimitFunction rateLimitFunction, + ScheduledExecutorService scheduledExecutorService) { this.rateLimitFunction = rateLimitFunction; update(publishRate); this.scheduledExecutorService = scheduledExecutorService; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java index 07632814378d0..c22ed41fc1533 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java @@ -164,7 +164,7 @@ public void testBrokerLevelPublishRateDynamicUpdate() throws Exception{ "" + rateInMsg)); Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get(); Assert.assertNotNull(topicRef); - PrecisPublishLimiter limiter = ((PrecisPublishLimiter) ((AbstractTopic) topicRef).topicPublishRateLimiter); + PrecisePublishLimiter limiter = ((PrecisePublishLimiter) ((AbstractTopic) topicRef).topicPublishRateLimiter); Awaitility.await().untilAsserted(() -> Assert.assertEquals(limiter.publishMaxMessageRate, rateInMsg)); Assert.assertEquals(limiter.publishMaxByteRate, 0); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java similarity index 57% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java index a0d0df52d2417..73cb43d52b112 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisPublishLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisePublishLimiterTest.java @@ -23,35 +23,35 @@ import org.apache.pulsar.common.policies.data.PublishRate; import org.testng.annotations.Test; -public class PrecisPublishLimiterTest { +public class PrecisePublishLimiterTest { @Test void shouldResetMsgLimitAfterUpdate() { - PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(), () -> { + PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(), () -> { }); - precisPublishLimiter.update(new PublishRate(1, 1)); - assertFalse(precisPublishLimiter.tryAcquire(99, 99)); - precisPublishLimiter.update(new PublishRate(-1, 100)); - assertTrue(precisPublishLimiter.tryAcquire(99, 99)); + precisePublishLimiter.update(new PublishRate(1, 1)); + assertFalse(precisePublishLimiter.tryAcquire(99, 99)); + precisePublishLimiter.update(new PublishRate(-1, 100)); + assertTrue(precisePublishLimiter.tryAcquire(99, 99)); } @Test void shouldResetBytesLimitAfterUpdate() { - PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(), () -> { + PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(), () -> { }); - precisPublishLimiter.update(new PublishRate(1, 1)); - assertFalse(precisPublishLimiter.tryAcquire(99, 99)); - precisPublishLimiter.update(new PublishRate(100, -1)); - assertTrue(precisPublishLimiter.tryAcquire(99, 99)); + precisePublishLimiter.update(new PublishRate(1, 1)); + assertFalse(precisePublishLimiter.tryAcquire(99, 99)); + precisePublishLimiter.update(new PublishRate(100, -1)); + assertTrue(precisePublishLimiter.tryAcquire(99, 99)); } @Test void shouldCloseResources() throws Exception { for (int i = 0; i < 20000; i++) { - PrecisPublishLimiter precisPublishLimiter = new PrecisPublishLimiter(new PublishRate(100, 100), () -> { + PrecisePublishLimiter precisePublishLimiter = new PrecisePublishLimiter(new PublishRate(100, 100), () -> { }); - precisPublishLimiter.tryAcquire(99, 99); - precisPublishLimiter.close(); + precisePublishLimiter.tryAcquire(99, 99); + precisePublishLimiter.close(); } } -} \ No newline at end of file +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java index 9d2bd49ba04bd..b934ced08c5db 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PublishRateLimiterTest.java @@ -39,7 +39,7 @@ public class PublishRateLimiterTest { private final PublishRate publishRate = new PublishRate(10, 100); private final PublishRate newPublishRate = new PublishRate(20, 200); - private PrecisPublishLimiter precisPublishLimiter; + private PrecisePublishLimiter precisePublishLimiter; private PublishRateLimiterImpl publishRateLimiter; @BeforeMethod @@ -47,7 +47,7 @@ public void setup() throws Exception { policies.publishMaxMessageRate = new HashMap<>(); policies.publishMaxMessageRate.put(CLUSTER_NAME, publishRate); - precisPublishLimiter = new PrecisPublishLimiter(policies, CLUSTER_NAME, () -> System.out.print("Refresh permit")); + precisePublishLimiter = new PrecisePublishLimiter(policies, CLUSTER_NAME, () -> System.out.print("Refresh permit")); publishRateLimiter = new PublishRateLimiterImpl(policies, CLUSTER_NAME); } @@ -88,23 +88,25 @@ public void testPublishRateLimiterImplUpdate() { @Test public void testPrecisePublishRateLimiterUpdate() { - assertFalse(precisPublishLimiter.tryAcquire(15, 150)); + assertFalse(precisePublishLimiter.tryAcquire(15, 150)); //update - precisPublishLimiter.update(newPublishRate); - assertTrue(precisPublishLimiter.tryAcquire(15, 150)); + precisePublishLimiter.update(newPublishRate); + assertTrue(precisePublishLimiter.tryAcquire(15, 150)); } @Test public void testPrecisePublishRateLimiterAcquire() throws Exception { - Class precisPublishLimiterClass = Class.forName("org.apache.pulsar.broker.service.PrecisPublishLimiter"); - Field topicPublishRateLimiterOnMessageField = precisPublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnMessage"); - Field topicPublishRateLimiterOnByteField = precisPublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnByte"); + Class precisePublishLimiterClass = Class.forName("org.apache.pulsar.broker.service.PrecisePublishLimiter"); + Field topicPublishRateLimiterOnMessageField = precisePublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnMessage"); + Field topicPublishRateLimiterOnByteField = precisePublishLimiterClass.getDeclaredField("topicPublishRateLimiterOnByte"); topicPublishRateLimiterOnMessageField.setAccessible(true); topicPublishRateLimiterOnByteField.setAccessible(true); - RateLimiter topicPublishRateLimiterOnMessage = (RateLimiter)topicPublishRateLimiterOnMessageField.get(precisPublishLimiter); - RateLimiter topicPublishRateLimiterOnByte = (RateLimiter)topicPublishRateLimiterOnByteField.get(precisPublishLimiter); + RateLimiter topicPublishRateLimiterOnMessage = (RateLimiter)topicPublishRateLimiterOnMessageField.get( + precisePublishLimiter); + RateLimiter topicPublishRateLimiterOnByte = (RateLimiter)topicPublishRateLimiterOnByteField.get( + precisePublishLimiter); Method renewTopicPublishRateLimiterOnMessageMethod = topicPublishRateLimiterOnMessage.getClass().getDeclaredMethod("renew", null); Method renewTopicPublishRateLimiterOnByteMethod = topicPublishRateLimiterOnByte.getClass().getDeclaredMethod("renew", null); @@ -112,7 +114,7 @@ public void testPrecisePublishRateLimiterAcquire() throws Exception { renewTopicPublishRateLimiterOnByteMethod.setAccessible(true); // running tryAcquire in order to lazyInit the renewTask - precisPublishLimiter.tryAcquire(1, 10); + precisePublishLimiter.tryAcquire(1, 10); Field onMessageRenewTaskField = topicPublishRateLimiterOnMessage.getClass().getDeclaredField("renewTask"); Field onByteRenewTaskField = topicPublishRateLimiterOnByte.getClass().getDeclaredField("renewTask"); @@ -129,30 +131,30 @@ public void testPrecisePublishRateLimiterAcquire() throws Exception { renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire not exceeded - assertTrue(precisPublishLimiter.tryAcquire(1, 10)); + assertTrue(precisePublishLimiter.tryAcquire(1, 10)); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire numOfMessages exceeded - assertFalse(precisPublishLimiter.tryAcquire(11, 100)); + assertFalse(precisePublishLimiter.tryAcquire(11, 100)); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire msgSizeInBytes exceeded - assertFalse(precisPublishLimiter.tryAcquire(10, 101)); + assertFalse(precisePublishLimiter.tryAcquire(10, 101)); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire exceeded exactly - assertFalse(precisPublishLimiter.tryAcquire(10, 100)); + assertFalse(precisePublishLimiter.tryAcquire(10, 100)); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); renewTopicPublishRateLimiterOnMessageMethod.invoke(topicPublishRateLimiterOnMessage); renewTopicPublishRateLimiterOnByteMethod.invoke(topicPublishRateLimiterOnByte); // tryAcquire not exceeded - assertTrue(precisPublishLimiter.tryAcquire(9, 99)); + assertTrue(precisePublishLimiter.tryAcquire(9, 99)); } } From 7d4d7a60ac346aab5469685270930462fd7ad3e4 Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 15 May 2023 10:47:00 +0800 Subject: [PATCH 098/494] Revert "[fix][client] Seek should be thread-safe (#20242)" This reverts commit bc1764f9ef71dd31e8cd61c7571e493442bc6395. (cherry picked from commit 7b54664c364a7360f30cd37f0aa989ab13a14af4) --- .../pulsar/client/impl/ConsumerImpl.java | 109 ++++++++---------- .../pulsar/client/impl/ConsumerImplTest.java | 27 +---- 2 files changed, 50 insertions(+), 86 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 6c64fe0406954..199e8a9ae71b4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -208,7 +208,6 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private final AtomicReference clientCnxUsedForConsumerRegistration = new AtomicReference<>(); private final List previousExceptions = new CopyOnWriteArrayList(); - static ConsumerImpl newConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, @@ -252,12 +251,10 @@ static ConsumerImpl newConsumerImpl(PulsarClientImpl client, } protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurationData conf, - ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, - boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, - MessageId startMessageId, - long startMessageRollbackDurationInSec, Schema schema, - ConsumerInterceptors interceptors, - boolean createTopicIfDoesNotExist) { + ExecutorProvider executorProvider, int partitionIndex, boolean hasParentConsumer, + boolean parentConsumerHasListener, CompletableFuture> subscribeFuture, MessageId startMessageId, + long startMessageRollbackDurationInSec, Schema schema, ConsumerInterceptors interceptors, + boolean createTopicIfDoesNotExist) { super(client, topic, conf, conf.getReceiverQueueSize(), executorProvider, subscribeFuture, schema, interceptors); this.consumerId = client.newConsumerId(); @@ -322,21 +319,21 @@ protected ConsumerImpl(PulsarClientImpl client, String topic, ConsumerConfigurat } this.connectionHandler = new ConnectionHandler(this, - new BackoffBuilder() - .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), - TimeUnit.NANOSECONDS) - .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) - .setMandatoryStop(0, TimeUnit.MILLISECONDS) - .create(), + new BackoffBuilder() + .setInitialTime(client.getConfiguration().getInitialBackoffIntervalNanos(), + TimeUnit.NANOSECONDS) + .setMax(client.getConfiguration().getMaxBackoffIntervalNanos(), TimeUnit.NANOSECONDS) + .setMandatoryStop(0, TimeUnit.MILLISECONDS) + .create(), this); this.topicName = TopicName.get(topic); if (this.topicName.isPersistent()) { this.acknowledgmentsGroupingTracker = - new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); + new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); } else { this.acknowledgmentsGroupingTracker = - NonPersistentAcknowledgmentGroupingTracker.of(); + NonPersistentAcknowledgmentGroupingTracker.of(); } if (conf.getDeadLetterPolicy() != null) { @@ -413,16 +410,16 @@ public CompletableFuture unsubscribeAsync() { log.error("[{}][{}] Failed to unsubscribe: {}", topic, subscription, e.getCause().getMessage()); setState(State.Ready); unsubscribeFuture.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("Failed to unsubscribe the subscription %s of topic %s", - topicName.toString(), subscription))); + PulsarClientException.wrap(e.getCause(), + String.format("Failed to unsubscribe the subscription %s of topic %s", + topicName.toString(), subscription))); return null; }); } else { unsubscribeFuture.completeExceptionally( - new PulsarClientException( - String.format("The client is not connected to the broker when unsubscribing the " - + "subscription %s of the topic %s", subscription, topicName.toString()))); + new PulsarClientException( + String.format("The client is not connected to the broker when unsubscribing the " + + "subscription %s of the topic %s", subscription, topicName.toString()))); } return unsubscribeFuture; } @@ -1403,7 +1400,7 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien } private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId, - MessageIdData messageId, ClientCnx cnx) { + MessageIdData messageId, ClientCnx cnx) { // Lazy task scheduling to expire incomplete chunk message if (expireTimeOfIncompleteChunkedMessageMillis > 0 && expireChunkMessageTaskScheduled.compareAndSet(false, @@ -1448,7 +1445,7 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m increaseAvailablePermits(cnx); if (expireTimeOfIncompleteChunkedMessageMillis > 0 && System.currentTimeMillis() > (msgMetadata.getPublishTime() - + expireTimeOfIncompleteChunkedMessageMillis)) { + + expireTimeOfIncompleteChunkedMessageMillis)) { doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null); } else { trackMessage(msgId); @@ -1639,7 +1636,7 @@ protected void trackMessage(Message msg) { } protected void trackMessage(MessageId messageId) { - trackMessage(messageId, 0); + trackMessage(messageId, 0); } protected void trackMessage(MessageId messageId, int redeliveryCount) { @@ -1780,7 +1777,7 @@ private ByteBuf decryptPayloadIfNeeded(MessageIdData messageId, int redeliveryCo } private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, - ClientCnx currentCnx, boolean checkMaxMessageSize) { + ClientCnx currentCnx, boolean checkMaxMessageSize) { CompressionType compressionType = msgMetadata.getCompression(); CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType); int uncompressedSize = msgMetadata.getUncompressedSize(); @@ -1833,7 +1830,7 @@ private void discardCorruptedMessage(MessageIdImpl messageId, ClientCnx currentC } private void discardCorruptedMessage(MessageIdData messageId, ClientCnx currentCnx, - ValidationError validationError) { + ValidationError validationError) { log.error("[{}][{}] Discarding corrupted message at {}:{}", topic, subscription, messageId.getLedgerId(), messageId.getEntryId()); discardMessage(messageId, currentCnx, validationError); @@ -2025,8 +2022,8 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) String originTopicNameStr = getOriginTopicNameStr(message); TypedMessageBuilder typedMessageBuilderNew = producerDLQ.newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())) - .value(message.getData()) - .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)); + .value(message.getData()) + .properties(getPropertiesMap(message, originMessageIdStr, originTopicNameStr)); if (message.hasKey()) { typedMessageBuilderNew.key(message.getKey()); } @@ -2055,7 +2052,7 @@ private CompletableFuture processPossibleToDLQ(MessageIdAdv messageId) } result.complete(false); return null; - }); + }); } }, internalPinnedExecutor).exceptionally(ex -> { log.error("Dead letter producer exception with topic: {}", deadLetterPolicy.getDeadLetterTopic(), ex); @@ -2154,15 +2151,9 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, final CompletableFuture seekFuture = new CompletableFuture<>(); ClientCnx cnx = cnx(); - if (!duringSeek.compareAndSet(false, true)) { - log.warn("[{}][{}] Attempting to seek operation that is already in progress, cancelling {}", - topic, subscription, seekBy); - seekFuture.cancel(true); - return seekFuture; - } - MessageIdAdv originSeekMessageId = seekMessageId; seekMessageId = (MessageIdAdv) seekId; + duringSeek.set(true); log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy); cnx.sendRequestWithId(seek, requestId).thenRun(() -> { @@ -2180,9 +2171,9 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, log.error("[{}][{}] Failed to reset subscription: {}", topic, subscription, e.getCause().getMessage()); seekFuture.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("Failed to seek the subscription %s of the topic %s to %s", - subscription, topicName.toString(), seekBy))); + PulsarClientException.wrap(e.getCause(), + String.format("Failed to seek the subscription %s of the topic %s to %s", + subscription, topicName.toString(), seekBy))); return null; }); return seekFuture; @@ -2194,7 +2185,7 @@ public CompletableFuture seekAsync(long timestamp) { return seekAsyncCheckState(seekBy).orElseGet(() -> { long requestId = client.newRequestId(); return seekAsyncInternal(requestId, Commands.newSeek(consumerId, requestId, timestamp), - MessageId.earliest, seekBy); + MessageId.earliest, seekBy); }); } @@ -2360,11 +2351,10 @@ public CompletableFuture> getLastMessageIdsAsync() { public CompletableFuture internalGetLastMessageIdAsync() { if (getState() == State.Closing || getState() == State.Closed) { return FutureUtil - .failedFuture(new PulsarClientException.AlreadyClosedException( - String.format("The consumer %s was already closed when the subscription %s of the topic %s " - + "getting the last message id", consumerName, subscription, - topicName.toString()))); - } + .failedFuture(new PulsarClientException.AlreadyClosedException( + String.format("The consumer %s was already closed when the subscription %s of the topic %s " + + "getting the last message id", consumerName, subscription, topicName.toString()))); + } AtomicLong opTimeoutMs = new AtomicLong(client.getConfiguration().getOperationTimeoutMs()); Backoff backoff = new BackoffBuilder() @@ -2386,12 +2376,11 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, if (isConnected() && cnx != null) { if (!Commands.peerSupportsGetLastMessageId(cnx.getRemoteEndpointProtocolVersion())) { future.completeExceptionally( - new PulsarClientException.NotSupportedException( - String.format( - "The command `GetLastMessageId` is not supported for the protocol version %d. " - + "The consumer is %s, topic %s, subscription %s", - cnx.getRemoteEndpointProtocolVersion(), - consumerName, topicName.toString(), subscription))); + new PulsarClientException.NotSupportedException( + String.format("The command `GetLastMessageId` is not supported for the protocol version %d. " + + "The consumer is %s, topic %s, subscription %s", + cnx.getRemoteEndpointProtocolVersion(), + consumerName, topicName.toString(), subscription))); return; } @@ -2410,31 +2399,31 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, } if (log.isDebugEnabled()) { log.debug("[{}][{}] Successfully getLastMessageId {}:{}", - topic, subscription, lastMessageId.getLedgerId(), lastMessageId.getEntryId()); + topic, subscription, lastMessageId.getLedgerId(), lastMessageId.getEntryId()); } MessageId lastMsgId = lastMessageId.getBatchIndex() <= 0 ? new MessageIdImpl(lastMessageId.getLedgerId(), - lastMessageId.getEntryId(), lastMessageId.getPartition()) + lastMessageId.getEntryId(), lastMessageId.getPartition()) : new BatchMessageIdImpl(lastMessageId.getLedgerId(), lastMessageId.getEntryId(), - lastMessageId.getPartition(), lastMessageId.getBatchIndex()); + lastMessageId.getPartition(), lastMessageId.getBatchIndex()); future.complete(new GetLastMessageIdResponse(lastMsgId, markDeletePosition)); }).exceptionally(e -> { log.error("[{}][{}] Failed getLastMessageId command", topic, subscription); future.completeExceptionally( - PulsarClientException.wrap(e.getCause(), - String.format("The subscription %s of the topic %s gets the last message id was failed", - subscription, topicName.toString()))); + PulsarClientException.wrap(e.getCause(), + String.format("The subscription %s of the topic %s gets the last message id was failed", + subscription, topicName.toString()))); return null; }); } else { long nextDelay = Math.min(backoff.next(), remainingTime.get()); if (nextDelay <= 0) { future.completeExceptionally( - new PulsarClientException.TimeoutException( - String.format("The subscription %s of the topic %s could not get the last message id " - + "withing configured timeout", subscription, topicName.toString()))); + new PulsarClientException.TimeoutException( + String.format("The subscription %s of the topic %s could not get the last message id " + + "withing configured timeout", subscription, topicName.toString()))); return; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java index ab11675f5434c..29d180f5f9a16 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java @@ -19,7 +19,6 @@ package org.apache.pulsar.client.impl; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -27,9 +26,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertTrue; -import io.netty.buffer.ByteBuf; + import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -262,26 +259,4 @@ public void testTopicPriorityLevel() { assertThat(consumer.getPriorityLevel()).isEqualTo(1); } - - @Test(invocationTimeOut = 1000) - public void testSeekAsyncInternal() { - // given - ClientCnx cnx = mock(ClientCnx.class); - CompletableFuture clientReq = new CompletableFuture<>(); - when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())).thenReturn(clientReq); - - consumer.setClientCnx(cnx); - consumer.setState(HandlerState.State.Ready); - - // when - CompletableFuture firstResult = consumer.seekAsync(1L); - CompletableFuture secondResult = consumer.seekAsync(1L); - - clientReq.complete(null); - - // then - assertTrue(firstResult.isDone()); - assertTrue(secondResult.isCancelled()); - verify(cnx, times(1)).sendRequestWithId(any(ByteBuf.class), anyLong()); - } } From 650d66c356f7d1addf1048051c4aa9ad1fa7eae0 Mon Sep 17 00:00:00 2001 From: tison Date: Mon, 15 May 2023 15:10:11 +0800 Subject: [PATCH 099/494] [fix][client] thread-safe seek Signed-off-by: tison (cherry picked from commit 0acf8f89d6057170e990b128936175fc5dd33be3) --- .../pulsar/client/impl/ConsumerImpl.java | 11 +++++++- .../pulsar/client/impl/ConsumerImplTest.java | 27 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 199e8a9ae71b4..2dd245c10f55b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2151,9 +2151,18 @@ private CompletableFuture seekAsyncInternal(long requestId, ByteBuf seek, final CompletableFuture seekFuture = new CompletableFuture<>(); ClientCnx cnx = cnx(); + if (!duringSeek.compareAndSet(false, true)) { + final String message = String.format( + "[%s][%s] attempting to seek operation that is already in progress (seek by %s)", + topic, subscription, seekBy); + log.warn("[{}][{}] Attempting to seek operation that is already in progress, cancelling {}", + topic, subscription, seekBy); + seekFuture.completeExceptionally(new IllegalStateException(message)); + return seekFuture; + } + MessageIdAdv originSeekMessageId = seekMessageId; seekMessageId = (MessageIdAdv) seekId; - duringSeek.set(true); log.info("[{}][{}] Seeking subscription to {}", topic, subscription, seekBy); cnx.sendRequestWithId(seek, requestId).thenRun(() -> { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java index 29d180f5f9a16..5a223d5da15c0 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerImplTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -26,7 +27,9 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; - +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; +import io.netty.buffer.ByteBuf; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutorService; @@ -259,4 +262,26 @@ public void testTopicPriorityLevel() { assertThat(consumer.getPriorityLevel()).isEqualTo(1); } + + @Test(invocationTimeOut = 1000) + public void testSeekAsyncInternal() { + // given + ClientCnx cnx = mock(ClientCnx.class); + CompletableFuture clientReq = new CompletableFuture<>(); + when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())).thenReturn(clientReq); + + consumer.setClientCnx(cnx); + consumer.setState(HandlerState.State.Ready); + + // when + CompletableFuture firstResult = consumer.seekAsync(1L); + CompletableFuture secondResult = consumer.seekAsync(1L); + + clientReq.complete(null); + + // then + assertTrue(firstResult.isDone()); + assertTrue(secondResult.isCompletedExceptionally()); + verify(cnx, times(1)).sendRequestWithId(any(ByteBuf.class), anyLong()); + } } From b3394540c218bb139fcc0658df2e6f4b4ac459da Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Tue, 16 May 2023 17:19:46 +0800 Subject: [PATCH 100/494] [fix][ml] Fix ledger left in OPEN state when enable `inactiveLedgerRollOverTimeMs` (#20276) close `currentLegder` after roll current ledger if full (cherry picked from commit 426ad3e53ba621c442bce89eb208add8f9d1563e) --- .../mledger/impl/ManagedLedgerImpl.java | 31 ++++++++++++++++--- .../mledger/impl/ManagedLedgerTest.java | 15 +++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index c74db884a95a9..15e9d332fa103 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1775,15 +1775,19 @@ public void closeComplete(int rc, LedgerHandle lh, Object o) { + "acked ledgerId %s", currentLedger.getId(), lh.getId()); if (rc == BKException.Code.OK) { - log.debug("Successfully closed ledger {}", lh.getId()); + if (log.isDebugEnabled()) { + log.debug("[{}] Successfully closed ledger {}, trigger by rollover full ledger", + name, lh.getId()); + } } else { - log.warn("Error when closing ledger {}. Status={}", lh.getId(), BKException.getMessage(rc)); + log.warn("[{}] Error when closing ledger {}, trigger by rollover full ledger, Status={}", + name, lh.getId(), BKException.getMessage(rc)); } ledgerClosed(lh); createLedgerAfterClosed(); } - }, System.nanoTime()); + }, null); } } @@ -4353,7 +4357,26 @@ public void checkInactiveLedgerAndRollOver() { long currentTimeMs = System.currentTimeMillis(); if (inactiveLedgerRollOverTimeMs > 0 && currentTimeMs > (lastAddEntryTimeMs + inactiveLedgerRollOverTimeMs)) { log.info("[{}] Closing inactive ledger, last-add entry {}", name, lastAddEntryTimeMs); - ledgerClosed(currentLedger); + if (STATE_UPDATER.compareAndSet(this, State.LedgerOpened, State.ClosingLedger)) { + LedgerHandle currentLedger = this.currentLedger; + currentLedger.asyncClose((rc, lh, o) -> { + checkArgument(currentLedger.getId() == lh.getId(), "ledgerId %s doesn't match with " + + "acked ledgerId %s", currentLedger.getId(), lh.getId()); + + if (rc == BKException.Code.OK) { + if (log.isDebugEnabled()) { + log.debug("[{}] Successfully closed ledger {}, trigger by inactive ledger check", + name, lh.getId()); + } + } else { + log.warn("[{}] Error when closing ledger {}, trigger by inactive ledger check, Status={}", + name, lh.getId(), BKException.getMessage(rc)); + } + + ledgerClosed(lh); + // we do not create ledger here, since topic is inactive for a long time. + }, null); + } } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index dd30cde72e769..70ddbb9998fd8 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -88,6 +88,7 @@ import org.apache.bookkeeper.client.PulsarMockBookKeeper; import org.apache.bookkeeper.client.PulsarMockLedgerHandle; import org.apache.bookkeeper.client.api.LedgerEntries; +import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -3859,12 +3860,26 @@ public void testInactiveLedgerRollOver() throws Exception { ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("rollover_inactive", config); ManagedCursor cursor = ledger.openCursor("c1"); + List ledgerIds = new ArrayList<>(); + int totalAddEntries = 5; for (int i = 0; i < totalAddEntries; i++) { String content = "entry"; // 5 bytes ledger.checkInactiveLedgerAndRollOver(); ledger.addEntry(content.getBytes()); Thread.sleep(inactiveLedgerRollOverTimeMs * 5); + + ledgerIds.add(ledger.currentLedger.getId()); + } + + Map ledgerMap = bkc.getLedgerMap(); + // skip check last ledger, it should be open + for (int i = 0; i < ledgerIds.size() - 1; i++) { + long ledgerId = ledgerIds.get(i); + LedgerMetadata ledgerMetadata = ledgerMap.get(ledgerId).getLedgerMetadata(); + if (ledgerMetadata != null) { + assertTrue(ledgerMetadata.isClosed()); + } } List ledgers = ledger.getLedgersInfoAsList(); From 7b74d89f6ae0f9117b955280da32d2a6619d4c93 Mon Sep 17 00:00:00 2001 From: zhangwd3 Date: Tue, 16 May 2023 11:23:11 +0800 Subject: [PATCH 101/494] correcting spelling mistakes Signed-off-by: zhangwd3 (cherry picked from commit ff58c6203ba6367f8ba293bc1a4e994725b5758a) --- .../org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java | 2 +- .../broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java | 2 +- .../apache/pulsar/client/api/SimpleProducerConsumerTest.java | 2 +- .../apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java | 2 +- .../src/main/java/org/apache/pulsar/client/api/Reader.java | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 42ef264b6db04..c9fce46a30515 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -118,7 +118,7 @@ public static double getCpuUsageForCGroup() { * *

* Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this - * far. Real CPU usage should equal the sum substracting the idle cycles, this would include iowait, irq and steal. + * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal. */ public static ResourceUsage getCpuUsageForEntireHost() { try (Stream stream = Files.lines(Paths.get(PROC_STAT_PATH))) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 318f37f7f7a97..8412658d86ae8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -155,7 +155,7 @@ private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) { * * * Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this - * far. Real CPU usage should equal the sum substracting the idle cycles, this would include iowait, irq and steal. + * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal. */ private double getTotalCpuUsageForEntireHost() { LinuxInfoUtils.ResourceUsage cpuUsageForEntireHost = getCpuUsageForEntireHost(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 1bc437195d96d..f3a00531eba56 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -3136,7 +3136,7 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe pulsarClient.newProducer().topic("persistent://my-property/my-ns/myenc-topic1") .addEncryptionKey("client-non-existant-rsa.pem").cryptoKeyReader(new EncKeyReader()) .create(); - Assert.fail("Producer creation should not suceed if failing to read key"); + Assert.fail("Producer creation should not succeed if failing to read key"); } catch (Exception e) { // ok } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java index 9f94d894632b8..e126d963a88f5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/v1/V1_ProducerConsumerTest.java @@ -2296,7 +2296,7 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe .addEncryptionKey("client-non-existant-rsa.pem") .cryptoKeyReader(new EncKeyReader()) .create(); - Assert.fail("Producer creation should not suceed if failing to read key"); + Assert.fail("Producer creation should not succeed if failing to read key"); } catch (Exception e) { // ok } diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java index 451b9fa638203..98fcdb453bb76 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/Reader.java @@ -36,14 +36,14 @@ public interface Reader extends Closeable { /** * Read the next message in the topic. * - * @return the next messasge + * @return the next message * @throws PulsarClientException */ Message readNext() throws PulsarClientException; /** * Read the next message in the topic waiting for a maximum of timeout - * time units. Returns null if no message is recieved in that time. + * time units. Returns null if no message is received in that time. * * @return the next message(Could be null if none received in time) * @throws PulsarClientException From 60d7b8f93a4be9310377eb1e0cc2f99cd71cd5af Mon Sep 17 00:00:00 2001 From: wangda <38549158+daziz@users.noreply.github.com> Date: Wed, 17 May 2023 19:43:42 +0800 Subject: [PATCH 102/494] [fix][doc] Correcting spelling mistakes (#20340) Signed-off-by: zhangwd3 Co-authored-by: tison (cherry picked from commit 1a66b640c3cd86bfca75dc9ab37bfdb37427a13f) --- .../java/org/apache/pulsar/client/impl/schema/SchemaUtils.java | 2 +- .../main/java/org/apache/pulsar/common/api/raw/RawMessage.java | 2 +- .../pulsar/functions/instance/state/BKStateStoreImpl.java | 2 +- .../functions/auth/KubernetesSecretsTokenAuthProvider.java | 2 +- .../apache/pulsar/functions/worker/rest/api/ComponentImpl.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java index 526b0c96be05b..8acbf26559b7b 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/client/impl/schema/SchemaUtils.java @@ -210,7 +210,7 @@ public static String jsonifySchemaInfo(SchemaInfo schemaInfo) { } /** - * Jsonify the schema info with verison. + * Jsonify the schema info with version. * * @param schemaInfoWithVersion the schema info * @return the jsonified schema info with version diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java index 8e2f51c4edbf0..a02208396fc91 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/api/raw/RawMessage.java @@ -102,7 +102,7 @@ public interface RawMessage { Optional getKey(); /** - * Get the schema verison of the message. + * Get the schema version of the message. * * @return the schema version of the message */ diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java index df1eae9b6a7ab..bf43f18b175e7 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/state/BKStateStoreImpl.java @@ -166,7 +166,7 @@ public CompletableFuture getAsync(String key) { data.readBytes(result); // Set position to off the buffer to the beginning, since the position after the // read is going to be end of the buffer - // If we do not rewind to the begining here, users will have to explicitly do + // If we do not rewind to the beginning here, users will have to explicitly do // this in their function code // in order to use any of the ByteBuffer operations result.position(0); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java index 916b8e9a6e1ad..e611801250f37 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/auth/KubernetesSecretsTokenAuthProvider.java @@ -161,7 +161,7 @@ public void cleanUpAuthData(Function.FunctionDetails funcDetails, Optional Date: Thu, 18 May 2023 11:15:53 +0800 Subject: [PATCH 103/494] [fix][broker]Fix deadlock of metadata store (#20189) Motivation: This task loadOrCreatePersistentTopic occupied the event thread of the ZK client so that other ZK tasks could not be finished anymore(Including the task itself), and it calls bundlesCache.synchronous().get(nsname) which is a blocking method. Modification: Since the method getBundle(topic) will eventually call the method bundlesCache.synchronous().get(nsname), use getBundleAsync(topic) instead of getBundle(topic) to avoid blocking the thread. (cherry picked from commit 4678c36d4023a2bb8361e0a70673b96de33f06ac) --- .../broker/namespace/NamespaceService.java | 32 +++++++++++-------- .../broker/namespace/OwnershipCache.java | 6 ++-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index a0e2bf7534c02..9d8d9e3890a19 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -38,7 +38,9 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Matcher; @@ -1137,16 +1139,17 @@ public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName) new IllegalArgumentException("Invalid class of NamespaceBundle: " + suName.getClass().getName())); } + /** + * @Deprecated This method is only used in test now. + */ + @Deprecated public boolean isServiceUnitActive(TopicName topicName) { try { - OwnedBundle ownedBundle = ownershipCache.getOwnedBundle(getBundle(topicName)); - if (ownedBundle == null) { - return false; - } - return ownedBundle.isActive(); - } catch (Exception e) { - LOG.warn("Unable to find OwnedBundle for topic - [{}]", topicName, e); - return false; + return isServiceUnitActiveAsync(topicName).get(pulsar.getConfig() + .getMetadataStoreOperationTimeoutSeconds(), SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + LOG.warn("Unable to find OwnedBundle for topic in time - [{}]", topicName, e); + throw new RuntimeException(e); } } @@ -1156,12 +1159,13 @@ public CompletableFuture isServiceUnitActiveAsync(TopicName topicName) return getBundleAsync(topicName) .thenCompose(bundle -> loadManager.get().checkOwnershipAsync(Optional.of(topicName), bundle)); } - Optional> res = ownershipCache.getOwnedBundleAsync(getBundle(topicName)); - if (!res.isPresent()) { - return CompletableFuture.completedFuture(false); - } - - return res.get().thenApply(ob -> ob != null && ob.isActive()); + return getBundleAsync(topicName).thenCompose(bundle -> { + Optional> optionalFuture = ownershipCache.getOwnedBundleAsync(bundle); + if (!optionalFuture.isPresent()) { + return CompletableFuture.completedFuture(false); + } + return optionalFuture.get().thenApply(ob -> ob != null && ob.isActive()); + }); } private boolean isNamespaceOwned(NamespaceName fqnn) throws Exception { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java index 7d0b5a4147721..86003153714cb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/OwnershipCache.java @@ -288,10 +288,10 @@ public Optional> getOwnedBundleAsync(NamespaceBun /** * Disable bundle in local cache and on zk. - * - * @param bundle - * @throws Exception + * @Deprecated This is a dangerous method which is currently only used for test, it will occupy the ZK thread. + * Please switch to your own thread after calling this method. */ + @Deprecated public CompletableFuture disableOwnership(NamespaceBundle bundle) { return updateBundleState(bundle, false) .thenCompose(__ -> { From a72fbfe8f3a5f938efe7111a46e74ea2716f4e45 Mon Sep 17 00:00:00 2001 From: Raghavender Mittapalli <14803749+syk-coder@users.noreply.github.com> Date: Thu, 18 May 2023 09:10:25 +0530 Subject: [PATCH 104/494] [fix][broker] Use user-specified bundle size on creating a namespace anti-affinity group with the default local policies (#20327) (cherry picked from commit 908d0b3f459167a3c3df2a85c23096df336427da) --- .../apache/pulsar/broker/admin/impl/NamespacesBase.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 8c72d0b0286d3..97029eb5ce128 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -1710,7 +1710,8 @@ protected String internalGetNamespaceAntiAffinityGroup() { try { return getLocalPolicies() .getLocalPolicies(namespaceName) - .orElse(new LocalPolicies()).namespaceAntiAffinityGroup; + .orElseGet(() -> new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles()) + , null, null)).namespaceAntiAffinityGroup; } catch (Exception e) { log.error("[{}] Failed to get the antiAffinityGroup of namespace {}", clientAppId(), namespaceName, e); throw new RestException(Status.NOT_FOUND, "Couldn't find namespace policies"); @@ -1760,7 +1761,9 @@ protected List internalGetAntiAffinityNamespaces(String cluster, String throw new RuntimeException(e); } - String storedAntiAffinityGroup = policies.orElse(new LocalPolicies()).namespaceAntiAffinityGroup; + String storedAntiAffinityGroup = policies.orElseGet(() -> + new LocalPolicies(getBundles(config().getDefaultNumberOfNamespaceBundles()), + null, null)).namespaceAntiAffinityGroup; return antiAffinityGroup.equalsIgnoreCase(storedAntiAffinityGroup); }).collect(Collectors.toList()); From c93d687316555be37628db62fdf6cdd252af754c Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 20 May 2023 03:38:24 +0800 Subject: [PATCH 105/494] [fix] [broker] In Key_Shared mode: remove unnecessary mechanisms of message skip to avoid unnecessary consumption stuck (#20335) ### Motivation - https://github.com/apache/pulsar/pull/7105 provide a mechanism to avoid a stuck consumer affecting the consumption of other consumers: - if all consumers can not accept more messages, stop delivering messages to the client. - if one consumer can not accept more messages, just read new messages and deliver them to other consumers. - https://github.com/apache/pulsar/pull/7553 provide a mechanism to fix the issue of lost order of consumption: If the consumer cannot accept any more messages, skip the consumer for the next round of message delivery because there may be messages with the same key in the replay queue. - https://github.com/apache/pulsar/pull/10762 provide a mechanism to fix the issue of lost order of consumption: If there have any messages with the same key in the replay queue, do not deliver the new messages to this consumer. https://github.com/apache/pulsar/pull/10762 and https://github.com/apache/pulsar/pull/7553 do the same thing and https://github.com/apache/pulsar/pull/10762 is better than https://github.com/apache/pulsar/pull/7553 , so https://github.com/apache/pulsar/pull/7553 is unnecessary. ### Modifications remove the mechanism provided by https://github.com/apache/pulsar/pull/7553 to avoid unnecessary consumption stuck. (cherry picked from commit 1e664b7f550ffa28d3c810f3b7d6d625d5905eb3) --- ...tStickyKeyDispatcherMultipleConsumers.java | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java index 1a8c6e180a2a2..8f05530f58bfa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumers.java @@ -71,17 +71,12 @@ public class PersistentStickyKeyDispatcherMultipleConsumers extends PersistentDi */ private final LinkedHashMap recentlyJoinedConsumers; - private final Set stuckConsumers; - private final Set nextStuckConsumers; - PersistentStickyKeyDispatcherMultipleConsumers(PersistentTopic topic, ManagedCursor cursor, Subscription subscription, ServiceConfiguration conf, KeySharedMeta ksm) { super(topic, cursor, subscription, ksm.isAllowOutOfOrderDelivery()); this.allowOutOfOrderDelivery = ksm.isAllowOutOfOrderDelivery(); this.recentlyJoinedConsumers = allowOutOfOrderDelivery ? null : new LinkedHashMap<>(); - this.stuckConsumers = new HashSet<>(); - this.nextStuckConsumers = new HashSet<>(); this.keySharedMode = ksm.getKeySharedMode(); switch (this.keySharedMode) { case AUTO_SPLIT: @@ -226,8 +221,6 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis } } - nextStuckConsumers.clear(); - final Map> groupedEntries = localGroupedEntries.get(); groupedEntries.clear(); final Map> consumerStickyKeyHashesMap = new HashMap<>(); @@ -318,14 +311,11 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis // acquire message-dispatch permits for already delivered messages acquirePermitsForDeliveredMessages(topic, cursor, totalEntries, totalMessagesSent, totalBytesSent); - stuckConsumers.clear(); - if (totalMessagesSent == 0 && (recentlyJoinedConsumers == null || recentlyJoinedConsumers.isEmpty())) { // This means, that all the messages we've just read cannot be dispatched right now. // This condition can only happen when: // 1. We have consumers ready to accept messages (otherwise the would not haven been triggered) // 2. All keys in the current set of messages are routing to consumers that are currently busy - // and stuck is not caused by stuckConsumers // // The solution here is to move on and read next batch of messages which might hopefully contain // also keys meant for other consumers. @@ -334,10 +324,7 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis // ahead in the stream while the new consumers are not ready to accept the new messages, // therefore would be most likely only increase the distance between read-position and mark-delete // position. - if (!nextStuckConsumers.isEmpty()) { - isDispatcherStuckOnReplays = true; - stuckConsumers.addAll(nextStuckConsumers); - } + isDispatcherStuckOnReplays = true; return true; } else if (currentThreadKeyNumber == 0) { return true; @@ -348,8 +335,6 @@ protected synchronized boolean trySendMessagesToConsumers(ReadType readType, Lis private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List entries, int maxMessages, ReadType readType, Set stickyKeyHashes) { if (maxMessages == 0) { - // the consumer was stuck - nextStuckConsumers.add(consumer); return 0; } if (readType == ReadType.Normal && stickyKeyHashes != null @@ -366,13 +351,6 @@ private int getRestrictedMaxEntriesForConsumer(Consumer consumer, List en // At this point, all the old messages were already consumed and this consumer // is now ready to receive any message if (maxReadPosition == null) { - // stop to dispatch by stuckConsumers - if (stuckConsumers.contains(consumer)) { - if (log.isDebugEnabled()) { - log.debug("[{}] stop to dispatch by stuckConsumers, consumer: {}", name, consumer); - } - return 0; - } // The consumer has not recently joined, so we can send all messages return maxMessages; } From 4cee3c3e987b04211f8e5c4495b9cb3a8bdfd4a6 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Sat, 20 May 2023 09:34:27 -0700 Subject: [PATCH 106/494] [fix][broker] managedLedger.getConfig().getProperties().putAll(properties) NPE (#20361) (cherry picked from commit aa7decc5b75894e864f3c5962c5cffad255abf25) --- .../apache/pulsar/broker/admin/impl/PersistentTopicsBase.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index de75c5f2132c5..5cd6ff8cbd955 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -699,7 +699,11 @@ private CompletableFuture internalUpdateNonPartitionedTopicProperties(Map< @Override public void updatePropertiesComplete(Map properties, Object ctx) { + if (managedLedger.getConfig().getProperties() == null) { + managedLedger.getConfig().setProperties(new HashMap<>()); + } managedLedger.getConfig().getProperties().putAll(properties); + future.complete(null); } From 5d34dadc2fbb6b82a97e3fa07b09beb5fa4b7be9 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Mon, 22 May 2023 08:51:10 +0800 Subject: [PATCH 107/494] [fix][broker] Fix broker load manager class filter NPE (#20350) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PIP: https://github.com/apache/pulsar/issues/16691 ### Motivation When upgrading the pulsar version and changing the pulsar load manager to `ExtensibleLoadManagerImpl` it might cause NPE. The root cause is the old version of pulsar does not contain the `loadManagerClassName` field. ``` 2023-05-18T05:42:50,557+0000 [pulsar-io-4-1] INFO org.apache.pulsar.broker.service.ServerCnx - [/127.0.0.6:51345] connected with role=[pulsarinstance-v3-0-n@test.dev](mailto:pulsarinstance-v3-0-n@test.dev) using authMethod=token, clientVersion=Pulsar Go 0.9.0, clientProtocolVersion=18, proxyVersion=null 2023-05-18T05:42:50,558+0000 [pulsar-io-4-1] WARN org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup [pulsarinstance-v3-0-n@test.dev](mailto:pulsarinstance-v3-0-n@test.dev) for topic persistent://xxx with error java.lang.NullPointerException: Cannot invoke “String.equals(Object)” because the return value of “org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData.getLoadManagerClassName()” is null java.util.concurrent.CompletionException: java.lang.NullPointerException: Cannot invoke “String.equals(Object)” because the return value of “org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData.getLoadManagerClassName()” is null at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315) ~[?:?] at java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1194) ~[?:?] at java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309) ~[?:?] at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.selectAsync(ExtensibleLoadManagerImpl.java:385) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1] at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.lambda$assign$6(ExtensibleLoadManagerImpl.java:336) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1] at java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1187) ~[?:?] at java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309) ~[?:?] at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.lambda$assign$10(ExtensibleLoadManagerImpl.java:333) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1] at org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap$Section.put(ConcurrentOpenHashMap.java:409) ~[io.streamnative-pulsar-common-3.0.0.1.jar:3.0.0.1] at org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap.computeIfAbsent(ConcurrentOpenHashMap.java:243) ~[io.streamnative-pulsar-common-3.0.0.1.jar:3.0.0.1] at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.assign(ExtensibleLoadManagerImpl.java:327) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1] at org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerWrapper.findBrokerServiceUrl(ExtensibleLoadManagerWrapper.java:66) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1] at org.apache.pulsar.broker.namespace.NamespaceService.lambda$getBrokerServiceUrlAsync$0(NamespaceService.java:191) ~[io.streamnative-pulsar-broker-3.0.0.1.jar:3.0.0.1] ``` ### Modifications * Add null check when using`getLoadManagerClassName`. * Add test to cover this case. * Add `RedirectManager` unit test. (cherry picked from commit b7f0004313ea4565717cc6d3c0b99aee5c079c6c) --- .../filter/BrokerLoadManagerClassFilter.java | 5 +- .../extensions/manager/RedirectManager.java | 17 ++- .../impl/BrokerLoadManagerClassFilter.java | 5 +- .../BrokerLoadManagerClassFilterTest.java | 3 +- .../manager/RedirectManagerTest.java | 111 ++++++++++++++++++ .../BrokerLoadManagerClassFilterTest.java | 6 + 6 files changed, 139 insertions(+), 8 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java index 4ee28a5225a0d..07109b277ae98 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; +import java.util.Objects; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; @@ -43,7 +44,9 @@ public Map filter( } brokers.entrySet().removeIf(entry -> { BrokerLookupData v = entry.getValue(); - return !v.getLoadManagerClassName().equals(context.brokerConfiguration().getLoadManagerClassName()); + // The load manager class name can be null if the cluster has old version of broker. + return !Objects.equals(v.getLoadManagerClassName(), + context.brokerConfiguration().getLoadManagerClassName()); }); return brokers; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java index 4aff77937a5b4..3455b333b0ae7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManager.java @@ -19,9 +19,11 @@ package org.apache.pulsar.broker.loadbalance.extensions.manager; import static org.apache.pulsar.broker.loadbalance.LoadManager.LOADBALANCE_BROKERS_ROOT; +import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -48,6 +50,12 @@ public RedirectManager(PulsarService pulsar) { this.brokerLookupDataLockManager = pulsar.getCoordinationService().getLockManager(BrokerLookupData.class); } + @VisibleForTesting + public RedirectManager(PulsarService pulsar, LockManager brokerLookupDataLockManager) { + this.pulsar = pulsar; + this.brokerLookupDataLockManager = brokerLookupDataLockManager; + } + public CompletableFuture> getAvailableBrokerLookupDataAsync() { return brokerLookupDataLockManager.listLocks(LOADBALANCE_BROKERS_ROOT).thenCompose(availableBrokers -> { Map map = new ConcurrentHashMap<>(); @@ -69,7 +77,7 @@ public CompletableFuture> getAvailableBrokerLookup public CompletableFuture> findRedirectLookupResultAsync() { String currentLMClassName = pulsar.getConfiguration().getLoadManagerClassName(); - boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfig(), log); + boolean debug = ExtensibleLoadManagerImpl.debug(pulsar.getConfiguration(), log); return getAvailableBrokerLookupDataAsync().thenApply(lookupDataMap -> { if (lookupDataMap.isEmpty()) { String errorMsg = "No available broker found."; @@ -89,9 +97,10 @@ public CompletableFuture> findRedirectLookupResultAsync() log.warn(errorMsg); throw new IllegalStateException(errorMsg); } - if (latestServiceLookupData.get().getLoadManagerClassName().equals(currentLMClassName)) { + + if (Objects.equals(latestServiceLookupData.get().getLoadManagerClassName(), currentLMClassName)) { if (debug) { - log.info("We don't need to redirect, current load manager class name: {}", + log.info("No need to redirect, current load manager class name: {}", currentLMClassName); } return Optional.empty(); @@ -99,7 +108,7 @@ public CompletableFuture> findRedirectLookupResultAsync() var serviceLookupDataObj = latestServiceLookupData.get(); var candidateBrokers = new ArrayList(); lookupDataMap.forEach((key, value) -> { - if (value.getLoadManagerClassName().equals(serviceLookupDataObj.getLoadManagerClassName())) { + if (Objects.equals(value.getLoadManagerClassName(), serviceLookupDataObj.getLoadManagerClassName())) { candidateBrokers.add(value); } }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java index 5d6a56ba86960..13e3fdc537e79 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilter.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import java.util.Objects; import java.util.Set; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilter; @@ -32,8 +33,8 @@ public void filter(Set brokers, BundleData bundleToAssign, LoadData loadData, ServiceConfiguration conf) throws BrokerFilterException { loadData.getBrokerData().forEach((key, value) -> { - if (!value.getLocalData().getLoadManagerClassName() - .equals(conf.getLoadManagerClassName())) { + // The load manager class name can be null if the cluster has old version of broker. + if (!Objects.equals(value.getLocalData().getLoadManagerClassName(), conf.getLoadManagerClassName())) { brokers.remove(key); } }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java index 0169b57fe993e..4aef87cf63aa8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java @@ -44,7 +44,8 @@ public void test() throws BrokerFilterException { "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), - "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()) + "broker4", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), + "broker5", getLookupData("3.0.0", null) ); Map result = filter.filter(new HashMap<>(originalBrokers), null, context); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java new file mode 100644 index 0000000000000..cbf77b59d5ad6 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/manager/RedirectManagerTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance.extensions.manager; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; +import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; +import org.apache.pulsar.broker.lookup.LookupResult; +import org.apache.pulsar.policies.data.loadbalancer.AdvertisedListener; +import org.testng.annotations.Test; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + + +/** + * Unit test {@link RedirectManager}. + */ +public class RedirectManagerTest { + + @Test + public void testFindRedirectLookupResultAsync() throws ExecutionException, InterruptedException { + PulsarService pulsar = mock(PulsarService.class); + ServiceConfiguration configuration = new ServiceConfiguration(); + when(pulsar.getConfiguration()).thenReturn(configuration); + RedirectManager redirectManager = spy(new RedirectManager(pulsar, null)); + + configuration.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + configuration.setLoadBalancerDebugModeEnabled(true); + + // Test 1: No load manager class name found. + doReturn(CompletableFuture.completedFuture( + new HashMap<>(){{ + put("broker-1", getLookupData("broker-1", null, 10)); + put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 1)); + }} + )).when(redirectManager).getAvailableBrokerLookupDataAsync(); + + // Should redirect to broker-1, since broker-1 has the latest load manager, even though the class name is null. + Optional lookupResult = redirectManager.findRedirectLookupResultAsync().get(); + assertTrue(lookupResult.isPresent()); + assertTrue(lookupResult.get().getLookupData().getBrokerUrl().contains("broker-1")); + + // Test 2: Should redirect to broker-1, since the latest broker are using ExtensibleLoadManagerImpl + doReturn(CompletableFuture.completedFuture( + new HashMap<>(){{ + put("broker-1", getLookupData("broker-1", ExtensibleLoadManagerImpl.class.getName(), 10)); + put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 1)); + }} + )).when(redirectManager).getAvailableBrokerLookupDataAsync(); + + lookupResult = redirectManager.findRedirectLookupResultAsync().get(); + assertTrue(lookupResult.isPresent()); + assertTrue(lookupResult.get().getLookupData().getBrokerUrl().contains("broker-1")); + + + // Test 3: Should not redirect, since current broker are using ModularLoadManagerImpl + doReturn(CompletableFuture.completedFuture( + new HashMap<>(){{ + put("broker-1", getLookupData("broker-1", ExtensibleLoadManagerImpl.class.getName(), 10)); + put("broker-2", getLookupData("broker-2", ModularLoadManagerImpl.class.getName(), 100)); + }} + )).when(redirectManager).getAvailableBrokerLookupDataAsync(); + + lookupResult = redirectManager.findRedirectLookupResultAsync().get(); + assertFalse(lookupResult.isPresent()); + } + + + public BrokerLookupData getLookupData(String broker, String loadManagerClassName, long startTimeStamp) { + String webServiceUrl = "http://" + broker + ":8080"; + String webServiceUrlTls = "https://" + broker + ":8081"; + String pulsarServiceUrl = "pulsar://" + broker + ":6650"; + String pulsarServiceUrlTls = "pulsar+ssl://" + broker + ":6651"; + Map advertisedListeners = new HashMap<>(); + Map protocols = new HashMap<>(){{ + put("kafka", "9092"); + }}; + return new BrokerLookupData( + webServiceUrl, webServiceUrlTls, pulsarServiceUrl, + pulsarServiceUrlTls, advertisedListeners, protocols, true, true, + loadManagerClassName, startTimeStamp, "3.0.0"); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java index 856bbac029226..56332111f935b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BrokerLoadManagerClassFilterTest.java @@ -46,8 +46,12 @@ public void test() throws BrokerFilterException { LocalBrokerData localBrokerData1 = new LocalBrokerData(); localBrokerData1.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + + LocalBrokerData localBrokerData2 = new LocalBrokerData(); + localBrokerData2.setLoadManagerClassName(null); loadData.getBrokerData().put("broker1", new BrokerData(localBrokerData)); loadData.getBrokerData().put("broker2", new BrokerData(localBrokerData1)); + loadData.getBrokerData().put("broker3", new BrokerData(localBrokerData2)); ServiceConfiguration conf = new ServiceConfiguration(); conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); @@ -55,6 +59,7 @@ public void test() throws BrokerFilterException { Set brokers = new HashSet<>(){{ add("broker1"); add("broker2"); + add("broker3"); }}; filter.filter(brokers, null, loadData, conf); @@ -64,6 +69,7 @@ public void test() throws BrokerFilterException { brokers = new HashSet<>(){{ add("broker1"); add("broker2"); + add("broker3"); }}; conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); filter.filter(brokers, null, loadData, conf); From 9733f7e94727b8f47070670f496e9534aa2e17c2 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Tue, 23 May 2023 21:26:49 -0700 Subject: [PATCH 108/494] [fix][broker] pre-create non-partitioned system topics for load balance extension (#20370) PIP: https://github.com/apache/pulsar/issues/16691 ### Motivation We need to create system topics without partitions explicitly. Currently, we do not support partitioned system topics. ### Modifications create system topics without partitions explicitly (cherry picked from commit 1080ad5c787bb317347b3f1f12b78ba3dec49757) --- .../extensions/ExtensibleLoadManagerImpl.java | 17 +++++++++++++++++ .../channel/ServiceUnitStateChannelImpl.java | 2 ++ .../ExtensibleLoadManagerImplTest.java | 2 ++ .../channel/ServiceUnitStateChannelTest.java | 2 ++ 4 files changed, 23 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 4ebf537f7a8a8..348098df87458 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -78,6 +78,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceName; @@ -212,6 +213,19 @@ public static boolean debug(ServiceConfiguration config, Logger log) { return config.isLoadBalancerDebugModeEnabled() || log.isDebugEnabled(); } + public static void createSystemTopic(PulsarService pulsar, String topic) throws PulsarServerException { + try { + pulsar.getAdminClient().topics().createNonPartitionedTopic(topic); + log.info("Created topic {}.", topic); + } catch (PulsarAdminException.ConflictException ex) { + if (debug(pulsar.getConfiguration(), log)) { + log.info("Topic {} already exists.", topic, ex); + } + } catch (PulsarAdminException e) { + throw new PulsarServerException(e); + } + } + @Override public void start() throws PulsarServerException { if (this.started) { @@ -246,6 +260,9 @@ public void start() throws PulsarServerException { this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies); this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper)); + createSystemTopic(pulsar, BROKER_LOAD_DATA_STORE_TOPIC); + createSystemTopic(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); + try { this.brokerLoadDataStore = LoadDataStoreFactory .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index b4c4e7fd5d42a..f476675d01bf6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -286,6 +286,8 @@ public synchronized void start() throws PulsarServerException { PulsarClusterMetadataSetup.createNamespaceIfAbsent (pulsar.getPulsarResources(), NamespaceName.SYSTEM_NAMESPACE, config.getClusterName()); + ExtensibleLoadManagerImpl.createSystemTopic(pulsar, TOPIC); + producer = pulsar.getClient().newProducer(schema) .enableBatching(true) .compressionType(MSG_COMPRESSION_TYPE) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index b71eeb4745b87..d72ce9661b988 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -97,6 +97,7 @@ import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; @@ -129,6 +130,7 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { @Override public void setup() throws Exception { conf.setForceDeleteNamespaceAllowed(true); + conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); conf.setAllowAutoTopicCreation(true); conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 77c80187a63e1..0f682cf048fe9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -86,6 +86,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.TableViewImpl; +import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.NotificationType; @@ -129,6 +130,7 @@ public class ServiceUnitStateChannelTest extends MockedPulsarServiceBaseTest { @Override protected void setup() throws Exception { conf.setAllowAutoTopicCreation(true); + conf.setAllowAutoTopicCreationType(TopicType.PARTITIONED); conf.setLoadBalancerDebugModeEnabled(true); conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10); super.internalSetup(conf); From 8b86f48699a05d57f27c70016cb2a81ca98c6a55 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 24 May 2023 16:41:52 +0800 Subject: [PATCH 109/494] [fix][fn] Fix JavaInstanceStarter inferring type class name error (#19896) (cherry picked from commit 05e57dd3a443c5b99c21054c56a1b497455fa867) --- .../runtime/JavaInstanceStarter.java | 19 ++++++++++---- .../runtime/thread/ThreadRuntime.java | 19 ++++++++------ .../runtime/thread/ThreadRuntimeFactory.java | 26 ++++++++++++++++--- 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index deff690815d9c..b78f07076f674 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -49,10 +49,13 @@ import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.proto.InstanceControlGrpc; +import org.apache.pulsar.functions.runtime.thread.ThreadRuntime; import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.secretsprovider.ClearTextSecretsProvider; import org.apache.pulsar.functions.secretsprovider.SecretsProvider; import org.apache.pulsar.functions.utils.FunctionCommon; +import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManager; +import org.apache.pulsar.functions.utils.functioncache.FunctionCacheManagerImpl; @Slf4j @@ -185,7 +188,10 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL functionDetailsJsonString = functionDetailsJsonString.substring(0, functionDetailsJsonString.length() - 1); } JsonFormat.parser().merge(functionDetailsJsonString, functionDetailsBuilder); - inferringMissingTypeClassName(functionDetailsBuilder, functionInstanceClassLoader); + FunctionCacheManager fnCache = new FunctionCacheManagerImpl(rootClassLoader); + ClassLoader functionClassLoader = ThreadRuntime.loadJars(jarFile, instanceConfig, functionId, + functionDetailsBuilder.getName(), narExtractionDirectory, fnCache); + inferringMissingTypeClassName(functionDetailsBuilder, functionClassLoader); Function.FunctionDetails functionDetails = functionDetailsBuilder.build(); instanceConfig.setFunctionDetails(functionDetails); instanceConfig.setPort(port); @@ -230,7 +236,7 @@ public void start(String[] args, ClassLoader functionInstanceClassLoader, ClassL .tlsHostnameVerificationEnable(isTrue(tlsHostNameVerificationEnabled)) .tlsTrustCertsFilePath(tlsTrustCertFilePath).build(), secretsProvider, collectorRegistry, narExtractionDirectory, rootClassLoader, - exposePulsarAdminClientEnabled, webServiceUrl); + exposePulsarAdminClientEnabled, webServiceUrl, fnCache); runtimeSpawner = new RuntimeSpawner( instanceConfig, jarFile, @@ -322,7 +328,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func Map userConfigs = new Gson().fromJson(functionDetailsBuilder.getUserConfig(), new TypeToken>() { }.getType()); - boolean isWindowConfigPresent = userConfigs.containsKey(WindowConfig.WINDOW_CONFIG_KEY); + boolean isWindowConfigPresent = + userConfigs != null && userConfigs.containsKey(WindowConfig.WINDOW_CONFIG_KEY); String className = functionDetailsBuilder.getClassName(); if (isWindowConfigPresent) { WindowConfig windowConfig = new Gson().fromJson( @@ -353,7 +360,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func case SINK: if ((functionDetailsBuilder.hasSink() && functionDetailsBuilder.getSink().getTypeClassName().isEmpty())) { - String typeArg = getSinkType(functionDetailsBuilder.getClassName(), classLoader).getName(); + String typeArg = + getSinkType(functionDetailsBuilder.getSink().getClassName(), classLoader).getName(); Function.SinkSpec.Builder sinkBuilder = Function.SinkSpec.newBuilder(functionDetailsBuilder.getSink()); @@ -371,7 +379,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func case SOURCE: if ((functionDetailsBuilder.hasSource() && functionDetailsBuilder.getSource().getTypeClassName().isEmpty())) { - String typeArg = getSourceType(functionDetailsBuilder.getClassName(), classLoader).getName(); + String typeArg = + getSourceType(functionDetailsBuilder.getSource().getClassName(), classLoader).getName(); Function.SourceSpec.Builder sourceBuilder = Function.SourceSpec.newBuilder(functionDetailsBuilder.getSource()); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java index 0aa0cd95aef09..ed128568bcf50 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java @@ -137,14 +137,16 @@ private static ClassLoader getFunctionClassLoader(InstanceConfig instanceConfig, .getClassLoader(); } } - return loadJars(jarFile, instanceConfig, functionId, narExtractionDirectory, fnCache); + return loadJars(jarFile, instanceConfig, functionId, instanceConfig.getFunctionDetails().getName(), + narExtractionDirectory, fnCache); } - private static ClassLoader loadJars(String jarFile, - InstanceConfig instanceConfig, - String functionId, - String narExtractionDirectory, - FunctionCacheManager fnCache) throws Exception { + public static ClassLoader loadJars(String jarFile, + InstanceConfig instanceConfig, + String functionId, + String functionName, + String narExtractionDirectory, + FunctionCacheManager fnCache) throws Exception { if (jarFile == null) { return Thread.currentThread().getContextClassLoader(); } @@ -175,8 +177,9 @@ private static ClassLoader loadJars(String jarFile, Collections.emptyList()); } - log.info("Initialize function class loader for function {} at function cache manager, functionClassLoader: {}", - instanceConfig.getFunctionDetails().getName(), fnCache.getClassLoader(functionId)); + log.info( + "Initialize function class loader for function {} at function cache manager, functionClassLoader: {}", + functionName, fnCache.getClassLoader(functionId)); fnClassLoader = fnCache.getClassLoader(functionId); if (null == fnClassLoader) { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java index 7bc055b25d6b9..cb9ad27a2dff8 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntimeFactory.java @@ -86,7 +86,21 @@ public ThreadRuntimeFactory(String threadGroupName, String pulsarServiceUrl, stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry, narExtractionDirectory, rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(), - Optional.empty()); + Optional.empty(), null); + } + + public ThreadRuntimeFactory(String threadGroupName, String pulsarServiceUrl, + String stateStorageImplClass, + String storageServiceUrl, + AuthenticationConfig authConfig, SecretsProvider secretsProvider, + FunctionCollectorRegistry collectorRegistry, String narExtractionDirectory, + ClassLoader rootClassLoader, boolean exposePulsarAdminClientEnabled, + String pulsarWebServiceUrl, FunctionCacheManager fnCache) throws Exception { + initialize(threadGroupName, Optional.empty(), pulsarServiceUrl, authConfig, + stateStorageImplClass, storageServiceUrl, null, secretsProvider, collectorRegistry, + narExtractionDirectory, + rootClassLoader, exposePulsarAdminClientEnabled, pulsarWebServiceUrl, Optional.empty(), + Optional.empty(), fnCache); } private void initialize(String threadGroupName, Optional memoryLimit, @@ -96,7 +110,7 @@ private void initialize(String threadGroupName, Optional connectorsManager, - Optional functionsManager) + Optional functionsManager, FunctionCacheManager fnCache) throws PulsarClientException { if (rootClassLoader == null) { @@ -106,7 +120,10 @@ private void initialize(String threadGroupName, Optional Date: Wed, 24 May 2023 21:56:30 +0800 Subject: [PATCH 110/494] [fix][broker] Invalidate metadata children cache after key deleted (#20363) (cherry picked from commit 7dcb3eab2916d42bb58fc1639dc58e11b5230997) --- .../org/apache/pulsar/metadata/impl/AbstractMetadataStore.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 072d513cca962..6fcf8eb6b49b2 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -328,6 +328,7 @@ public void accept(Notification n) { if (type == NotificationType.Created || type == NotificationType.Deleted) { existsCache.synchronous().invalidate(path); + childrenCache.synchronous().invalidate(path); String parent = parent(path); if (parent != null) { childrenCache.synchronous().invalidate(parent); @@ -385,6 +386,7 @@ private CompletableFuture deleteInternal(String path, Optional expec // Ensure caches are invalidated before the operation is confirmed return storeDelete(path, expectedVersion).thenRun(() -> { existsCache.synchronous().invalidate(path); + childrenCache.synchronous().invalidate(path); String parent = parent(path); if (parent != null) { childrenCache.synchronous().invalidate(parent); From 698529b69eed0fea319341bcf19430115db41577 Mon Sep 17 00:00:00 2001 From: houxiaoyu Date: Mon, 29 May 2023 10:20:08 +0800 Subject: [PATCH 111/494] [fix][broker] Fix ledger cachemiss size metric (#20257) (cherry picked from commit 1c813fdaeb9f752b84347b5a62e219dd66e79a26) --- .../bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java | 4 ++-- .../bookkeeper/mledger/impl/cache/EntryCacheDisabled.java | 4 ++-- .../bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java index cb3d72cc5972f..e057dee99538e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java @@ -100,8 +100,8 @@ public void recordReadEntriesError() { readEntriesOpsFailed.recordEvent(); } - public void recordReadEntriesOpsCacheMisses() { - readEntriesOpsCacheMisses.recordEvent(); + public void recordReadEntriesOpsCacheMisses(int count, long totalSize) { + readEntriesOpsCacheMisses.recordMultipleEvents(count, totalSize); } public void addAddEntryLatencySample(long latency, TimeUnit unit) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java index d2add99b701ac..d1050e0062826 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/EntryCacheDisabled.java @@ -93,7 +93,7 @@ public void asyncReadEntry(ReadHandle lh, long firstEntry, long lastEntry, boole } finally { ledgerEntries.close(); } - ml.getMbean().recordReadEntriesOpsCacheMisses(); + ml.getMbean().recordReadEntriesOpsCacheMisses(entries.size(), totalSize); ml.getFactory().getMbean().recordCacheMiss(entries.size(), totalSize); ml.getMbean().addReadEntriesSample(entries.size(), totalSize); @@ -121,7 +121,7 @@ public void asyncReadEntry(ReadHandle lh, PositionImpl position, AsyncCallbacks. LedgerEntry ledgerEntry = iterator.next(); EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor); - ml.getMbean().recordReadEntriesOpsCacheMisses(); + ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength()); ml.getFactory().getMbean().recordCacheMiss(1, returnEntry.getLength()); ml.getMbean().addReadEntriesSample(1, returnEntry.getLength()); callback.readEntryComplete(returnEntry, ctx); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java index 7747f9bcd93b6..27aec6f178e39 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/cache/RangeEntryCacheImpl.java @@ -256,7 +256,7 @@ private void asyncReadEntry0(ReadHandle lh, PositionImpl position, final ReadEnt LedgerEntry ledgerEntry = iterator.next(); EntryImpl returnEntry = RangeEntryCacheManagerImpl.create(ledgerEntry, interceptor); - ml.getMbean().recordReadEntriesOpsCacheMisses(); + ml.getMbean().recordReadEntriesOpsCacheMisses(1, returnEntry.getLength()); manager.mlFactoryMBean.recordCacheMiss(1, returnEntry.getLength()); ml.getMbean().addReadEntriesSample(1, returnEntry.getLength()); callback.readEntryComplete(returnEntry, ctx); @@ -450,7 +450,7 @@ CompletableFuture> readFromStorage(ReadHandle lh, } } - ml.getMbean().recordReadEntriesOpsCacheMisses(); + ml.getMbean().recordReadEntriesOpsCacheMisses(entriesToReturn.size(), totalSize); manager.mlFactoryMBean.recordCacheMiss(entriesToReturn.size(), totalSize); ml.getMbean().addReadEntriesSample(entriesToReturn.size(), totalSize); From 34bfe843eddc7acf26ce046c5f565b0108b1e17d Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 29 May 2023 14:46:38 +0800 Subject: [PATCH 112/494] [fix][broker] If ledger lost, cursor mark delete position can not forward (#18620) Motivation: Configuration `autoSkipNonRecoverableData` is designed to turn this feature on if we can accept partial data loss. When a ledger is lost, the broker will still work. But now we have this problem: If a ledger is lost, consumer and producer can work, but the cursor mark delete position can not forward. Modifications: - When an unrecoverable ledger is found, remove the records in`individualDeletedMessages` and `batchDeletedIndexes`. - When the managed cursor is recovered, check whether there are invalid records in `individualDeletedMessages` and `batchDeletedIndexes` and print a warning log. (cherry picked from commit ab810f4f59dd7d6c8b5313ceb334873a7d2cde31) --- .../bookkeeper/mledger/ManagedCursor.java | 6 + .../bookkeeper/mledger/ManagedLedger.java | 6 + .../mledger/impl/ManagedCursorImpl.java | 40 +++ .../mledger/impl/ManagedLedgerImpl.java | 7 + .../bookkeeper/mledger/impl/OpReadEntry.java | 5 + .../LedgerLostAndSkipNonRecoverableTest.java | 296 ++++++++++++++++++ 6 files changed, 360 insertions(+) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index 7802ed07781ba..edbfa0b43204e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -786,6 +786,12 @@ Set asyncReplayEntries( */ long getEstimatedSizeSinceMarkDeletePosition(); + /** + * If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is + * used to delete information about this ledger in the ManagedCursor. + */ + default void skipNonRecoverableLedger(long ledgerId){} + /** * Returns cursor throttle mark-delete rate. * diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java index 4ca56508891a1..c7dd8ea9129b7 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java @@ -631,6 +631,12 @@ void asyncSetProperties(Map properties, AsyncCallbacks.UpdatePro */ void trimConsumedLedgersInBackground(CompletableFuture promise); + /** + * If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is + * used to delete information about this ledger in the ManagedCursor. + */ + default void skipNonRecoverableLedger(long ledgerId){} + /** * Roll current ledger if it is full. */ diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index ef607fa7ed7cf..663081c932052 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -2718,6 +2718,46 @@ void setReadPosition(Position newReadPositionInt) { } } + /** + * Manually acknowledge all entries in the lost ledger. + * - Since this is an uncommon event, we focus on maintainability. So we do not modify + * {@link #individualDeletedMessages} and {@link #batchDeletedIndexes}, but call + * {@link #asyncDelete(Position, AsyncCallbacks.DeleteCallback, Object)}. + * - This method is valid regardless of the consumer ACK type. + * - If there is a consumer ack request after this event, it will also work. + */ + @Override + public void skipNonRecoverableLedger(final long ledgerId){ + LedgerInfo ledgerInfo = ledger.getLedgersInfo().get(ledgerId); + if (ledgerInfo == null) { + return; + } + lock.writeLock().lock(); + log.warn("[{}] [{}] Since the ledger [{}] is lost and the autoSkipNonRecoverableData is true, this ledger will" + + " be auto acknowledge in subscription", ledger.getName(), name, ledgerId); + try { + for (int i = 0; i < ledgerInfo.getEntries(); i++) { + if (!individualDeletedMessages.contains(ledgerId, i)) { + asyncDelete(PositionImpl.get(ledgerId, i), new AsyncCallbacks.DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + // ignore. + } + + @Override + public void deleteFailed(ManagedLedgerException ex, Object ctx) { + // The method internalMarkDelete already handled the failure operation. We only need to + // make sure the memory state is updated. + // If the broker crashed, the non-recoverable ledger will be detected again. + } + }, null); + } + } + } finally { + lock.writeLock().unlock(); + } + } + // ////////////////////////////////////////////////// void startCreatingNewMetadataLedger() { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 15e9d332fa103..9b3d7e46aaa8c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1741,6 +1741,13 @@ synchronized void ledgerClosed(final LedgerHandle lh) { } } + @Override + public void skipNonRecoverableLedger(long ledgerId){ + for (ManagedCursor managedCursor : cursors) { + managedCursor.skipNonRecoverableLedger(ledgerId); + } + } + synchronized void createLedgerAfterClosed() { if (isNeededCreateNewLedgerAfterCloseLedger()) { log.info("[{}] Creating a new ledger after closed", name); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java index 19211553a5f74..7b59c3903d5bc 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/OpReadEntry.java @@ -116,9 +116,11 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { readPosition, exception.getMessage()); final ManagedLedgerImpl ledger = (ManagedLedgerImpl) cursor.getManagedLedger(); Position nexReadPosition; + Long lostLedger = null; if (exception instanceof ManagedLedgerException.LedgerNotExistException) { // try to find and move to next valid ledger nexReadPosition = cursor.getNextLedgerPosition(readPosition.getLedgerId()); + lostLedger = readPosition.ledgerId; } else { // Skip this read operation nexReadPosition = ledger.getValidPositionAfterSkippedEntries(readPosition, count); @@ -131,6 +133,9 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { return; } updateReadPosition(nexReadPosition); + if (lostLedger != null) { + cursor.getManagedLedger().skipNonRecoverableLedger(lostLedger); + } checkReadCompletion(); } else { if (!(exception instanceof TooManyRequestsException)) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java new file mode 100644 index 0000000000000..389af8f2cd9f9 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/LedgerLostAndSkipNonRecoverableTest.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class LedgerLostAndSkipNonRecoverableTest extends ProducerConsumerBase { + + private static final String DEFAULT_NAMESPACE = "my-property/my-ns"; + + @BeforeClass + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + protected void doInitConf() throws Exception { + conf.setAutoSkipNonRecoverableData(true); + } + + @DataProvider(name = "batchEnabled") + public Object[][] batchEnabled(){ + return new Object[][]{ + {true}, + {false} + }; + } + + @Test(timeOut = 30000, dataProvider = "batchEnabled") + public void testMarkDeletedPositionCanForwardAfterTopicLedgerLost(boolean enabledBatch) throws Exception { + String topicSimpleName = UUID.randomUUID().toString().replaceAll("-", ""); + String subName = UUID.randomUUID().toString().replaceAll("-", ""); + String topicName = String.format("persistent://%s/%s", DEFAULT_NAMESPACE, topicSimpleName); + + log.info("create topic and subscription."); + Consumer sub = createConsumer(topicName, subName, enabledBatch); + sub.redeliverUnacknowledgedMessages(); + sub.close(); + + log.info("send many messages."); + int ledgerCount = 3; + int messageCountPerLedger = enabledBatch ? 25 : 5; + int messageCountPerEntry = enabledBatch ? 5 : 1; + List[] sendMessages = + sendManyMessages(topicName, ledgerCount, messageCountPerLedger, messageCountPerEntry); + int sendMessageCount = Arrays.asList(sendMessages).stream() + .flatMap(s -> s.stream()).collect(Collectors.toList()).size(); + log.info("send {} messages", sendMessageCount); + + log.info("make individual ack."); + ConsumerAndReceivedMessages consumerAndReceivedMessages1 = + waitConsumeAndAllMessages(topicName, subName, enabledBatch,false); + List[] messageIds = consumerAndReceivedMessages1.messageIds; + Consumer consumer = consumerAndReceivedMessages1.consumer; + MessageIdImpl individualPosition = messageIds[1].get(messageCountPerEntry - 1); + MessageIdImpl expectedMarkDeletedPosition = + new MessageIdImpl(messageIds[0].get(0).getLedgerId(), messageIds[0].get(0).getEntryId(), -1); + MessageIdImpl lastPosition = + new MessageIdImpl(messageIds[2].get(4).getLedgerId(), messageIds[2].get(4).getEntryId(), -1); + consumer.acknowledge(individualPosition); + consumer.acknowledge(expectedMarkDeletedPosition); + waitPersistentCursorLedger(topicName, subName, expectedMarkDeletedPosition.getLedgerId(), + expectedMarkDeletedPosition.getEntryId()); + consumer.close(); + + log.info("Make lost ledger [{}].", individualPosition.getLedgerId()); + pulsar.getBrokerService().getTopic(topicName, false).get().get().close(false); + pulsarTestContext.getMockBookKeeper().deleteLedger(individualPosition.getLedgerId()); + + log.info("send some messages."); + sendManyMessages(topicName, 3, messageCountPerEntry); + + log.info("receive all messages then verify mark deleted position"); + ConsumerAndReceivedMessages consumerAndReceivedMessages2 = + waitConsumeAndAllMessages(topicName, subName, enabledBatch, true); + waitMarkDeleteLargeAndEquals(topicName, subName, lastPosition.getLedgerId(), lastPosition.getEntryId()); + + // cleanup + consumerAndReceivedMessages2.consumer.close(); + admin.topics().delete(topicName); + } + + private ManagedCursorImpl getCursor(String topicName, String subName) throws Exception { + PersistentSubscription subscription_ = + (PersistentSubscription) pulsar.getBrokerService().getTopic(topicName, false) + .get().get().getSubscription(subName); + return (ManagedCursorImpl) subscription_.getCursor(); + } + + private void waitMarkDeleteLargeAndEquals(String topicName, String subName, final long markDeletedLedgerId, + final long markDeletedEntryId) throws Exception { + Awaitility.await().atMost(Duration.ofSeconds(45)).untilAsserted(() -> { + Position persistentMarkDeletedPosition = getCursor(topicName, subName).getMarkDeletedPosition(); + log.info("markDeletedPosition {}:{}, expected {}:{}", persistentMarkDeletedPosition.getLedgerId(), + persistentMarkDeletedPosition.getEntryId(), markDeletedLedgerId, markDeletedEntryId); + Assert.assertTrue(persistentMarkDeletedPosition.getLedgerId() >= markDeletedLedgerId); + if (persistentMarkDeletedPosition.getLedgerId() > markDeletedLedgerId){ + return; + } + Assert.assertTrue(persistentMarkDeletedPosition.getEntryId() >= markDeletedEntryId); + }); + } + + private void waitPersistentCursorLedger(String topicName, String subName, final long markDeletedLedgerId, + final long markDeletedEntryId) throws Exception { + Awaitility.await().untilAsserted(() -> { + Position persistentMarkDeletedPosition = getCursor(topicName, subName).getPersistentMarkDeletedPosition(); + Assert.assertEquals(persistentMarkDeletedPosition.getLedgerId(), markDeletedLedgerId); + Assert.assertEquals(persistentMarkDeletedPosition.getEntryId(), markDeletedEntryId); + }); + } + + private List[] sendManyMessages(String topicName, int ledgerCount, int messageCountPerLedger, + int messageCountPerEntry) throws Exception { + List[] messageIds = new List[ledgerCount]; + for (int i = 0; i < ledgerCount; i++){ + admin.topics().unload(topicName); + if (messageCountPerEntry == 1) { + messageIds[i] = sendManyMessages(topicName, messageCountPerLedger); + } else { + messageIds[i] = sendManyBatchedMessages(topicName, messageCountPerEntry, + messageCountPerLedger / messageCountPerEntry); + } + } + return messageIds; + } + + private List sendManyMessages(String topicName, int messageCountPerLedger, + int messageCountPerEntry) throws Exception { + if (messageCountPerEntry == 1) { + return sendManyMessages(topicName, messageCountPerLedger); + } else { + return sendManyBatchedMessages(topicName, messageCountPerEntry, + messageCountPerLedger / messageCountPerEntry); + } + } + + private List sendManyMessages(String topicName, int msgCount) throws Exception { + List messageIdList = new ArrayList<>(); + final Producer producer = pulsarClient.newProducer(Schema.JSON(String.class)) + .topic(topicName) + .enableBatching(false) + .create(); + long timestamp = System.currentTimeMillis(); + for (int i = 0; i < msgCount; i++){ + String messageSuffix = String.format("%s-%s", timestamp, i); + MessageIdImpl messageIdSent = (MessageIdImpl) producer.newMessage() + .key(String.format("Key-%s", messageSuffix)) + .value(String.format("Msg-%s", messageSuffix)) + .send(); + messageIdList.add(messageIdSent); + } + producer.close(); + return messageIdList; + } + + private List sendManyBatchedMessages(String topicName, int msgCountPerEntry, int entryCount) + throws Exception { + Producer producer = pulsarClient.newProducer(Schema.JSON(String.class)) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(Integer.MAX_VALUE, TimeUnit.SECONDS) + .batchingMaxMessages(Integer.MAX_VALUE) + .create(); + List> messageIdFutures = new ArrayList<>(); + for (int i = 0; i < entryCount; i++){ + for (int j = 0; j < msgCountPerEntry; j++){ + CompletableFuture messageIdFuture = + producer.newMessage().value(String.format("entry-seq[%s], batch_index[%s]", i, j)).sendAsync(); + messageIdFutures.add(messageIdFuture); + } + producer.flush(); + } + producer.close(); + FutureUtil.waitForAll(messageIdFutures).get(); + return messageIdFutures.stream().map(f -> (MessageIdImpl)f.join()).collect(Collectors.toList()); + } + + private ConsumerAndReceivedMessages waitConsumeAndAllMessages(String topicName, String subName, + final boolean enabledBatch, + boolean ack) throws Exception { + List messageIds = new ArrayList<>(); + final Consumer consumer = createConsumer(topicName, subName, enabledBatch); + while (true){ + Message message = consumer.receive(5, TimeUnit.SECONDS); + if (message != null){ + messageIds.add((MessageIdImpl) message.getMessageId()); + if (ack) { + consumer.acknowledge(message); + } + } else { + break; + } + } + log.info("receive {} messages", messageIds.size()); + return new ConsumerAndReceivedMessages(consumer, sortMessageId(messageIds, enabledBatch)); + } + + @AllArgsConstructor + private static class ConsumerAndReceivedMessages { + private Consumer consumer; + private List[] messageIds; + } + + private List[] sortMessageId(List messageIds, boolean enabledBatch){ + Map> map = messageIds.stream().collect(Collectors.groupingBy(v -> v.getLedgerId())); + TreeMap> sortedMap = new TreeMap<>(map); + List[] res = new List[sortedMap.size()]; + Iterator>> iterator = sortedMap.entrySet().iterator(); + for (int i = 0; i < sortedMap.size(); i++){ + res[i] = iterator.next().getValue(); + } + for (List list : res){ + list.sort((m1, m2) -> { + if (enabledBatch){ + BatchMessageIdImpl mb1 = (BatchMessageIdImpl) m1; + BatchMessageIdImpl mb2 = (BatchMessageIdImpl) m2; + return (int) (mb1.getLedgerId() * 1000000 + mb1.getEntryId() * 1000 + mb1.getBatchIndex() - + mb2.getLedgerId() * 1000000 + mb2.getEntryId() * 1000 + mb2.getBatchIndex()); + } + return (int) (m1.getLedgerId() * 1000 + m1.getEntryId() - + m2.getLedgerId() * 1000 + m2.getEntryId()); + }); + } + return res; + } + + private Consumer createConsumer(String topicName, String subName, boolean enabledBatch) throws Exception { + final Consumer consumer = pulsarClient.newConsumer(Schema.JSON(String.class)) + .autoScaledReceiverQueueSizeEnabled(false) + .subscriptionType(SubscriptionType.Failover) + .isAckReceiptEnabled(true) + .enableBatchIndexAcknowledgment(enabledBatch) + .receiverQueueSize(1000) + .topic(topicName) + .subscriptionName(subName) + .subscribe(); + return consumer; + } +} From d09664af2d8dbdd681db8a7ad1b76ffde5675314 Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Tue, 30 May 2023 10:07:09 +0800 Subject: [PATCH 113/494] [Fix][Txn] Unwrap the completion exception. (#20396) (cherry picked from commit 7ea8741af265bd8554e7a52a29a4ed59cb749dea) --- .../pulsar/broker/TransactionMetadataStoreService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java index 3e3b044ec51b8..35aa7cc2fdd26 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/TransactionMetadataStoreService.java @@ -169,7 +169,8 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc tcLoadSemaphore.release(); })).exceptionally(e -> { internalPinnedExecutor.execute(() -> { - completableFuture.completeExceptionally(e.getCause()); + Throwable realCause = FutureUtil.unwrapCompletionException(e); + completableFuture.completeExceptionally(realCause); // release before handle request queue, //in order to client reconnect infinite loop tcLoadSemaphore.release(); @@ -180,7 +181,7 @@ public CompletableFuture handleTcClientConnect(TransactionCoordinatorID tc CompletableFuture future = deque.poll(); if (future != null) { // this means that this tc client connection connect fail - future.completeExceptionally(e); + future.completeExceptionally(realCause); } else { break; } From 1ce9cca45e5479aeb5b3d69afb93b8e9c41378a8 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Tue, 30 May 2023 17:57:08 +0800 Subject: [PATCH 114/494] [fix][broker] Skip loading broker interceptor when disableBrokerInterceptors is true (#20422) Signed-off-by: Zixuan Liu (cherry picked from commit 639c460a585c2f6cbb6133849348c8b92d2f6fbd) --- .../pulsar/broker/ServiceConfiguration.java | 2 +- .../broker/intercept/BrokerInterceptors.java | 5 ++++ .../intercept/BrokerInterceptorTest.java | 25 +++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 2c48310f96482..8c85b9f28ab7e 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1353,7 +1353,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_SERVER, - doc = "Enable or disable the broker interceptor, which is only used for testing for now" + doc = "Enable or disable the broker interceptor" ) private boolean disableBrokerInterceptors = true; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java index cef3f0eb609a1..4ffd8732db94e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java @@ -59,6 +59,11 @@ public BrokerInterceptors(Map intercep * @return the collection of broker event interceptor */ public static BrokerInterceptor load(ServiceConfiguration conf) throws IOException { + if (conf.isDisableBrokerInterceptors()) { + log.info("Skip loading the broker interceptors when disableBrokerInterceptors is true"); + return null; + } + BrokerInterceptorDefinitions definitions = BrokerInterceptorUtils.searchForInterceptors(conf.getBrokerInterceptorsDirectory(), conf.getNarExtractionDirectory()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java index d1cf91635f992..c612104f8bf1b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java @@ -20,9 +20,12 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; + import java.io.IOException; import java.util.HashMap; import java.util.HashSet; @@ -36,6 +39,7 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -307,4 +311,25 @@ public void requestInterceptorFailedTest() { } } + @Test + public void testLoadWhenDisableBrokerInterceptorsIsTrue() throws IOException { + ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); + serviceConfiguration.setDisableBrokerInterceptors(true); + BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration); + assertNull(brokerInterceptor); + + verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors(); + verify(serviceConfiguration, times(0)).getBrokerInterceptorsDirectory(); + } + + @Test + public void testLoadWhenDisableBrokerInterceptorsIsFalse() throws IOException { + ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); + serviceConfiguration.setDisableBrokerInterceptors(false); + BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration); + assertNull(brokerInterceptor); + + verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors(); + verify(serviceConfiguration, times(1)).getBrokerInterceptorsDirectory(); + } } From b97a1b4c297d4a9e7e4de61a82050b59d0f0a8e9 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 30 May 2023 18:07:31 +0800 Subject: [PATCH 115/494] [fix] [meta]Switch to the metadata store thread after zk operation (#20303) (cherry picked from commit 35d961242089a0448cb425da637a2afbdaca0005) --- .../metadata/impl/AbstractMetadataStore.java | 13 ++++ .../pulsar/metadata/impl/ZKMetadataStore.java | 46 ++++++------ .../pulsar/metadata/MetadataStoreTest.java | 70 +++++++++++++++++++ 3 files changed, 106 insertions(+), 23 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 6fcf8eb6b49b2..4cadf2397a7fa 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -534,6 +535,18 @@ public void execute(Runnable task, CompletableFuture future) { } } + /** + * Run the task in the executor thread and fail the future if the executor is shutting down. + */ + @VisibleForTesting + public void execute(Runnable task, Supplier>> futures) { + try { + executor.execute(task); + } catch (final Throwable t) { + futures.get().forEach(f -> f.completeExceptionally(t)); + } + } + protected static String parent(String path) { int idx = path.lastIndexOf('/'); if (idx <= 0) { diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java index a6d8eb8344c96..079ae3e2ae5c3 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java @@ -204,29 +204,29 @@ protected void batchOperation(List ops) { } // Trigger all the futures in the batch - for (int i = 0; i < ops.size(); i++) { - OpResult opr = results.get(i); - MetadataOp op = ops.get(i); - - switch (op.getType()) { - case PUT: - handlePutResult(op.asPut(), opr); - break; - case DELETE: - handleDeleteResult(op.asDelete(), opr); - break; - case GET: - handleGetResult(op.asGet(), opr); - break; - case GET_CHILDREN: - handleGetChildrenResult(op.asGetChildren(), opr); - break; - - default: - op.getFuture().completeExceptionally(new MetadataStoreException( - "Operation type not supported in multi: " + op.getType())); - } - } + execute(() -> { + for (int i = 0; i < ops.size(); i++) { + OpResult opr = results.get(i); + MetadataOp op = ops.get(i); + switch (op.getType()) { + case PUT: + handlePutResult(op.asPut(), opr); + break; + case DELETE: + handleDeleteResult(op.asDelete(), opr); + break; + case GET: + handleGetResult(op.asGet(), opr); + break; + case GET_CHILDREN: + handleGetChildrenResult(op.asGetChildren(), opr); + break; + default: + op.getFuture().completeExceptionally(new MetadataStoreException( + "Operation type not supported in multi: " + op.getType())); + } + } + }, () -> ops.stream().map(MetadataOp::getFuture).collect(Collectors.toList())); }, null); } catch (Throwable t) { ops.forEach(o -> o.getFuture().completeExceptionally(new MetadataStoreException(t))); diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java index 949b4a9b2bacb..c87e9bda436bf 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -50,8 +51,10 @@ import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.Stat; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.assertj.core.util.Lists; import org.awaitility.Awaitility; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Slf4j @@ -425,6 +428,73 @@ public void testDeleteUnusedDirectories(String provider, Supplier urlSup assertFalse(store.exists(prefix).join()); } + @DataProvider(name = "conditionOfSwitchThread") + public Object[][] conditionOfSwitchThread(){ + return new Object[][]{ + {false, false}, + {false, true}, + {true, false}, + {true, true} + }; + } + + @Test(dataProvider = "conditionOfSwitchThread") + public void testThreadSwitchOfZkMetadataStore(boolean hasSynchronizer, boolean enabledBatch) throws Exception { + final String prefix = newKey(); + final String metadataStoreName = UUID.randomUUID().toString().replaceAll("-", ""); + MetadataStoreConfig.MetadataStoreConfigBuilder builder = + MetadataStoreConfig.builder().metadataStoreName(metadataStoreName); + builder.fsyncEnable(false); + builder.batchingEnabled(enabledBatch); + if (!hasSynchronizer) { + builder.synchronizer(null); + } + MetadataStoreConfig config = builder.build(); + @Cleanup + ZKMetadataStore store = (ZKMetadataStore) MetadataStoreFactory.create(zks.getConnectionString(), config); + + final Runnable verify = () -> { + String currentThreadName = Thread.currentThread().getName(); + String errorMessage = String.format("Expect to switch to thread %s, but currently it is thread %s", + metadataStoreName, currentThreadName); + assertTrue(Thread.currentThread().getName().startsWith(metadataStoreName), errorMessage); + }; + + // put with node which has parent(but the parent node is not exists). + store.put(prefix + "/a1/b1/c1", "value".getBytes(), Optional.of(-1L)).thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // put. + store.put(prefix + "/b1", "value".getBytes(), Optional.of(-1L)).thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // get. + store.get(prefix + "/b1").thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // get the node which is not exists. + store.get(prefix + "/non").thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // delete. + store.delete(prefix + "/b1", Optional.empty()).thenApply((ignore) -> { + verify.run(); + return null; + }).join(); + // delete the node which is not exists. + store.delete(prefix + "/non", Optional.empty()).thenApply((ignore) -> { + verify.run(); + return null; + }).exceptionally(ex -> { + verify.run(); + return null; + }).join(); + } + @Test(dataProvider = "impl") public void testPersistent(String provider, Supplier urlSupplier) throws Exception { String metadataUrl = urlSupplier.get(); From d4b55f45362089005a2259857d23c2f690f23bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Mikl=C3=B3s?= Date: Thu, 18 May 2023 17:23:55 +0200 Subject: [PATCH 116/494] [improve][fn] Use functions classloader in TopicSchema.newSchemaInstance() to fix ClassNotFoundException when using custom SerDe classes. (targeted for master) (#20115) (cherry picked from commit 43a989862f548fa3f67708a5fff62eb764af878c) --- .../pulsar/functions/instance/ContextImpl.java | 2 +- .../apache/pulsar/functions/sink/PulsarSink.java | 2 +- .../pulsar/functions/source/PulsarSource.java | 6 +++--- .../source/SingleConsumerPulsarSource.java | 11 ++++------- .../pulsar/functions/source/TopicSchema.java | 15 ++++++++++++--- .../pulsar/functions/source/TopicSchemaTest.java | 8 +++++--- .../functions/PulsarFunctionsTest.java | 8 ++++---- .../functions/PulsarFunctionsTestBase.java | 2 +- .../functions/java/PulsarFunctionsJavaTest.java | 11 +++++++---- .../java/PulsarWorkerRebalanceDrainTest.java | 15 +++++++++------ .../functions/utils/CommandGenerator.java | 10 +++++----- 11 files changed, 52 insertions(+), 38 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java index d64c5f9b52db6..5cbbcad24c7a2 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/ContextImpl.java @@ -148,7 +148,7 @@ public ContextImpl(InstanceConfig config, Logger logger, PulsarClient client, this.clientBuilder = clientBuilder; this.client = client; this.pulsarAdmin = pulsarAdmin; - this.topicSchema = new TopicSchema(client); + this.topicSchema = new TopicSchema(client, Thread.currentThread().getContextClassLoader()); this.statsManager = statsManager; this.producerBuilder = (ProducerBuilderImpl) client.newProducer().blockIfQueueFull(true).enableBatching(true) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 8add0a78c5fff..97a0ad0a2ce17 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -349,7 +349,7 @@ public PulsarSink(PulsarClient client, PulsarSinkConfig pulsarSinkConfig, Map buildPulsarSourceConsumerConfig(String t Class typeArg) { PulsarSourceConsumerConfig.PulsarSourceConsumerConfigBuilder consumerConfBuilder = PulsarSourceConsumerConfig.builder().isRegexPattern(conf.isRegexPattern()) - .receiverQueueSize(conf.getReceiverQueueSize()) - .consumerProperties(conf.getConsumerProperties()); + .receiverQueueSize(conf.getReceiverQueueSize()) + .consumerProperties(conf.getConsumerProperties()); Schema schema; if (conf.getSerdeClassName() != null && !conf.getSerdeClassName().isEmpty()) { diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java index 426723804cad1..d4d3ea00b9317 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/SingleConsumerPulsarSource.java @@ -44,14 +44,12 @@ public class SingleConsumerPulsarSource extends PulsarSource { private Consumer consumer; private final List> inputConsumers = new LinkedList<>(); - public SingleConsumerPulsarSource(PulsarClient pulsarClient, - SingleConsumerPulsarSourceConfig pulsarSourceConfig, - Map properties, - ClassLoader functionClassLoader) { + public SingleConsumerPulsarSource(PulsarClient pulsarClient, SingleConsumerPulsarSourceConfig pulsarSourceConfig, + Map properties, ClassLoader functionClassLoader) { super(pulsarClient, pulsarSourceConfig, properties, functionClassLoader); this.pulsarClient = pulsarClient; this.pulsarSourceConfig = pulsarSourceConfig; - this.topicSchema = new TopicSchema(pulsarClient); + this.topicSchema = new TopicSchema(pulsarClient, functionClassLoader); this.properties = properties; this.functionClassLoader = functionClassLoader; } @@ -60,8 +58,7 @@ public SingleConsumerPulsarSource(PulsarClient pulsarClient, public void open(Map config, SourceContext sourceContext) throws Exception { log.info("Opening pulsar source with config: {}", pulsarSourceConfig); - Class typeArg = Reflections.loadClass(this.pulsarSourceConfig.getTypeClassName(), - this.functionClassLoader); + Class typeArg = Reflections.loadClass(this.pulsarSourceConfig.getTypeClassName(), this.functionClassLoader); checkArgument(!Void.class.equals(typeArg), "Input type of Pulsar Function cannot be Void"); diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java index 57f49fed0cac9..a912bc7ecf103 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/source/TopicSchema.java @@ -19,7 +19,11 @@ package org.apache.pulsar.functions.source; import io.netty.buffer.ByteBuf; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.ByteBuffer; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -49,8 +53,13 @@ public class TopicSchema { private final Map> cachedSchemas = new HashMap<>(); private final PulsarClient client; - public TopicSchema(PulsarClient client) { + private final ClassLoader functionsClassloader; + + public TopicSchema(PulsarClient client, ClassLoader functionsClassloader) { this.client = client; + this.functionsClassloader = AccessController.doPrivileged( + (PrivilegedAction) () -> new URLClassLoader(new URL[0], functionsClassloader) + ); } /** @@ -244,11 +253,11 @@ private Schema newSchemaInstance(String topic, Class clazz, ConsumerCo @SuppressWarnings("unchecked") private Schema newSchemaInstance(String topic, Class clazz, String schemaTypeOrClassName, boolean input) { return newSchemaInstance(topic, clazz, new ConsumerConfig(schemaTypeOrClassName), input, - Thread.currentThread().getContextClassLoader()); + functionsClassloader); } @SuppressWarnings("unchecked") private Schema newSchemaInstance(String topic, Class clazz, ConsumerConfig conf, boolean input) { - return newSchemaInstance(topic, clazz, conf, input, Thread.currentThread().getContextClassLoader()); + return newSchemaInstance(topic, clazz, conf, input, functionsClassloader); } } diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java index 506ef67ab0740..c4e77cb3ff85d 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/TopicSchemaTest.java @@ -18,8 +18,6 @@ */ package org.apache.pulsar.functions.source; -import static org.testng.Assert.assertEquals; -import java.util.Optional; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -30,12 +28,16 @@ import org.apache.pulsar.functions.proto.Request; import org.testng.annotations.Test; +import java.util.Optional; + +import static org.testng.Assert.assertEquals; + @Slf4j public class TopicSchemaTest { @Test public void testGetSchema() { - TopicSchema topicSchema = new TopicSchema(null); + TopicSchema topicSchema = new TopicSchema(null, Thread.currentThread().getContextClassLoader()); String TOPIC = "public/default/test"; Schema schema = topicSchema.getSchema(TOPIC + "1", DummyClass.class, Optional.of(SchemaType.JSON)); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 6088628aac52b..1e54764ad5d2b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -24,10 +24,10 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.swagger.util.Json; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -43,8 +43,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - -import io.swagger.util.Json; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -77,8 +75,8 @@ import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.api.examples.AutoSchemaFunction; import org.apache.pulsar.functions.api.examples.AvroSchemaTestFunction; -import org.apache.pulsar.functions.api.examples.MergeTopicFunction; import org.apache.pulsar.functions.api.examples.InitializableFunction; +import org.apache.pulsar.functions.api.examples.MergeTopicFunction; import org.apache.pulsar.functions.api.examples.RecordFunction; import org.apache.pulsar.functions.api.examples.pojo.AvroTestObject; import org.apache.pulsar.functions.api.examples.pojo.Users; @@ -902,6 +900,7 @@ protected void submitFunction(Runtime runtime, String functionName, String functionFile, String functionClass, + Map inputSerdeClassNames, String outputSerdeClassName, Map userConfigs) throws Exception { @@ -916,6 +915,7 @@ protected void submitFunction(Runtime runtime, } generator.setSinkTopic(outputTopicName); generator.setFunctionName(functionName); + generator.setCustomSerDeSourceTopics(inputSerdeClassNames); generator.setOutputSerDe(outputSerdeClassName); if (userConfigs != null) { generator.setUserConfig(userConfigs); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java index 62aa36da1b6be..288ced63ae5eb 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTestBase.java @@ -54,7 +54,7 @@ public abstract class PulsarFunctionsTestBase extends PulsarTestSuite { public static final String SERDE_JAVA_CLASS = "org.apache.pulsar.functions.api.examples.CustomBaseToBaseFunction"; - public static final String SERDE_OUTPUT_CLASS = + public static final String SERDE_CLASS = "org.apache.pulsar.functions.api.examples.CustomBaseSerde"; public static final String EXCLAMATION_PYTHON_CLASS = diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java index 097a452937d27..939d6e19d1fb1 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarFunctionsJavaTest.java @@ -19,9 +19,9 @@ package org.apache.pulsar.tests.integration.functions.java; import static org.testng.Assert.assertEquals; - import java.util.Collections; - +import java.util.Map; +import org.apache.commons.collections4.map.HashedMap; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.policies.data.FunctionStatus; import org.apache.pulsar.common.policies.data.FunctionStatusUtil; @@ -70,10 +70,13 @@ private void testCustomSerdeFunction() throws Exception { admin.topics().createNonPartitionedTopic(outputTopicName); } + Map inputTopicsSerde = new HashedMap<>(); + inputTopicsSerde.put(inputTopicName, SERDE_CLASS); + String functionName = "test-serde-fn-" + randomName(8); submitFunction( - Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, - SERDE_OUTPUT_CLASS, Collections.singletonMap("serde-topic", outputTopicName) + Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, inputTopicsSerde, + SERDE_CLASS, Collections.singletonMap("serde-topic", outputTopicName) ); // get function info diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java index a6f54349f2689..de29cbc94c295 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/java/PulsarWorkerRebalanceDrainTest.java @@ -18,6 +18,9 @@ */ package org.apache.pulsar.tests.integration.functions.java; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import com.fasterxml.jackson.databind.MappingIterator; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -26,10 +29,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - -import com.fasterxml.jackson.databind.MappingIterator; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.commons.collections4.map.HashedMap; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.FunctionStatus; @@ -41,8 +43,6 @@ import org.apache.pulsar.tests.integration.topologies.FunctionRuntimeType; import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testng.annotations.Test; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; @Slf4j public abstract class PulsarWorkerRebalanceDrainTest extends PulsarFunctionsTest { @@ -251,9 +251,12 @@ private void createFunctionWorker(String functionName, String topicPrefix) throw admin.topics().createNonPartitionedTopic(outputTopicName); } + Map inputTopicsSerde = new HashedMap<>(); + inputTopicsSerde.put(inputTopicName, SERDE_CLASS); + submitFunction( - Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, - SERDE_OUTPUT_CLASS, Collections.singletonMap(topicPrefix, outputTopicName) + Runtime.JAVA, inputTopicName, outputTopicName, functionName, null, SERDE_JAVA_CLASS, inputTopicsSerde, + SERDE_CLASS, Collections.singletonMap(topicPrefix, outputTopicName) ); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java index 90fac7a055268..adc791fab4dc8 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/utils/CommandGenerator.java @@ -46,7 +46,7 @@ public enum Runtime { private String functionClassName; private String sourceTopic; private String sourceTopicPattern; - private Map customSereSourceTopics; + private Map customSerDeSourceTopics; private String sinkTopic; private String logTopic; private String outputSerDe; @@ -182,8 +182,8 @@ public String generateCreateFunctionCommand(String codeFile) { if (batchBuilder != null) { commandBuilder.append("--batch-builder" + batchBuilder); } - if (customSereSourceTopics != null && !customSereSourceTopics.isEmpty()) { - commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSereSourceTopics) + "\'"); + if (customSerDeSourceTopics != null && !customSerDeSourceTopics.isEmpty()) { + commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSerDeSourceTopics) + "\'"); } if (sinkTopic != null) { commandBuilder.append(" --output " + sinkTopic); @@ -280,8 +280,8 @@ public String generateUpdateFunctionCommand(String codeFile) { if (StringUtils.isNotEmpty(sourceTopic)) { commandBuilder.append(" --inputs " + sourceTopic); } - if (customSereSourceTopics != null && !customSereSourceTopics.isEmpty()) { - commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSereSourceTopics) + "\'"); + if (customSerDeSourceTopics != null && !customSerDeSourceTopics.isEmpty()) { + commandBuilder.append(" --customSerdeInputs \'" + new Gson().toJson(customSerDeSourceTopics) + "\'"); } if (batchBuilder != null) { commandBuilder.append("--batch-builder" + batchBuilder); From 22115c394541e4f6f4890d429bcca455ebd0d1ec Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Sun, 21 May 2023 17:50:44 -0700 Subject: [PATCH 117/494] [improve][broker] Gracefully shut down load balancer extension (#20315) ### Motivation Load balancer extension needs to shut down gracefully, especially when shutting down the leader broker. When the leader broker closes the leader election service too late, service unit lookups to the leader broker could fail during the shutdown. This could delay client reconnection time. Lookup failure logs ``` (shutdown starts) pulsar-broker-8 pulsar-broker 2023-04-22T00:19:52,630+0000 [pulsar-service-shutdown] INFO org.apache.pulsar.broker.PulsarService - Closing PulsarService pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,690+0000 [pulsar-io-4-5] INFO org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [pulsar-18-9] Closed connection [id: 0x907e74b0, L:/10.0.13.6:35838 ! R:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local/10.0.18.18:6650] -- Will try again in 0.1 s pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,691+0000 [pulsar-io-4-5] INFO org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [reader-30e6b40dd9] Closed connection [id: 0x907e74b0, L:/10.0.13.6:35838 ! R:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local/10.0.18.18:6650] -- Will try again in 0.1 s (znode is gone) pulsar-broker-8 pulsar-broker 2023-04-22T00:19:52,652+0000 [pulsar-load-manager-1-1] INFO org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl - BrokerRegistry detected the broker:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 registry has been deleted. (lookup failure) org.apache.pulsar.client.impl.ConnectionHandler - [persistent://pulsar/system/loadbalancer-service-unit-state] [reader-30e6b40dd9] Could not get connection to broker: org.apache.pulsar.client.api.PulsarClientException$ConnectException: Disconnected from server at pulsar-broker-3.pulsar-broker.pulsar.svc.cluster.local/10.0.13.6:6650 -- Will try again in 0.193 s pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,827+0000 [main-EventThread] INFO org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup null for topic persistent://pulsar/system/loadbalancer-service-unit-state with error Failed to look up a broker registry:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 for bundle:pulsar/system/0x40000000_0x50000000 pulsar-broker-3 pulsar-broker 2023-04-22T00:19:52,827+0000 [main-EventThread] INFO org.apache.pulsar.broker.lookup.TopicLookupBase - Failed to lookup null for topic persistent://pulsar/system/loadbalancer-service-unit-state with error Failed to look up a broker registry:pulsar-broker-8.pulsar-broker.pulsar.svc.cluster.local:8080 for bundle:pulsar/system/0x40000000_0x50000000 (leader election service has been closed) pulsar-broker-3 pulsar-broker 2023-04-22T00:19:55,569+0000 [pulsar-load-manager-1-1] INFO org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl - This broker:pulsar-broker-3.pulsar-broker.pulsar.svc.cluster.local:8080 plays the leader now. ``` ### Modifications During the shutdown flow, when calling loadbalancer.disableBroker(), the load balancer extension gracefully gives up the owned bundles to other brokers. After that, it closes the leader election service and removes its register in the metadata store. --- .../apache/pulsar/broker/PulsarService.java | 22 +++- .../extensions/ExtensibleLoadManagerImpl.java | 17 +++ .../ExtensibleLoadManagerWrapper.java | 2 +- .../channel/ServiceUnitStateChannel.java | 5 + .../channel/ServiceUnitStateChannelImpl.java | 112 +++++++++++++++--- .../ExtensibleLoadManagerImplTest.java | 45 +++++++ .../channel/ServiceUnitStateChannelTest.java | 65 ++++++---- 7 files changed, 222 insertions(+), 46 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index f833674ceeb0d..62d4634fa2d62 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -382,6 +382,17 @@ public void closeMetadataServiceSession() throws Exception { localMetadataStore.close(); } + private void closeLeaderElectionService() throws Exception { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + ExtensibleLoadManagerImpl.get(loadManager.get()).getLeaderElectionService().close(); + } else { + if (this.leaderElectionService != null) { + this.leaderElectionService.close(); + this.leaderElectionService = null; + } + } + } + @Override public void close() throws PulsarServerException { try { @@ -502,10 +513,7 @@ public CompletableFuture closeAsync() { this.bkClientFactory = null; } - if (this.leaderElectionService != null) { - this.leaderElectionService.close(); - this.leaderElectionService = null; - } + closeLeaderElectionService(); if (adminClient != null) { adminClient.close(); @@ -1316,7 +1324,11 @@ public boolean isRunning() { * @return a reference of the current LeaderElectionService instance. */ public LeaderElectionService getLeaderElectionService() { - return this.leaderElectionService; + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + return ExtensibleLoadManagerImpl.get(loadManager.get()).getLeaderElectionService(); + } else { + return this.leaderElectionService; + } } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 348098df87458..531ab18938a1e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -26,6 +26,7 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -397,12 +398,22 @@ public CompletableFuture> assign(Optional> selectAsync(ServiceUnitId bundle) { + return selectAsync(bundle, Collections.emptySet()); + } + + public CompletableFuture> selectAsync(ServiceUnitId bundle, + Set excludeBrokerSet) { BrokerRegistry brokerRegistry = getBrokerRegistry(); return brokerRegistry.getAvailableBrokerLookupDataAsync() .thenCompose(availableBrokers -> { LoadManagerContext context = this.getContext(); Map availableBrokerCandidates = new HashMap<>(availableBrokers); + if (!excludeBrokerSet.isEmpty()) { + for (String exclude : excludeBrokerSet) { + availableBrokerCandidates.remove(exclude); + } + } // Filter out brokers that do not meet the rules. List filterPipeline = getBrokerFilterPipeline(); @@ -702,4 +713,10 @@ private void monitor() { log.error("Failed to get the channel ownership.", e); } } + + public void disableBroker() throws Exception { + serviceUnitStateChannel.cleanOwnerships(); + leaderElectionService.close(); + brokerRegistry.unregister(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java index 1eabbe620e213..18e949537dedb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerWrapper.java @@ -74,7 +74,7 @@ public CompletableFuture checkOwnershipAsync(Optional to @Override public void disableBroker() throws Exception { - this.loadManager.getBrokerRegistry().unregister(); + this.loadManager.disableBroker(); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java index 4782a31fe0c56..6e75fe91a914f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannel.java @@ -206,4 +206,9 @@ public interface ServiceUnitStateChannel extends Closeable { * Cancels the ownership monitor. */ void cancelOwnershipMonitor(); + + /** + * Cleans the service unit ownerships from the current broker's channel. + */ + void cleanOwnerships(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index f476675d01bf6..489a00851057b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -91,7 +91,6 @@ import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceBundles; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.Metrics; @@ -110,6 +109,10 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { public static final CompressionType MSG_COMPRESSION_TYPE = CompressionType.ZSTD; private static final long MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS = 30 * 1000; // 30sec + + private static final int OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS = 5000; + private static final int OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS = 100; + private static final int OWNERSHIP_CLEAN_UP_CONVERGENCE_DELAY_IN_MILLIS = 3000; public static final long VERSION_ID_INIT = 1; // initial versionId private static final long OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS = 60; public static final long MAX_CLEAN_UP_DELAY_TIME_IN_SECS = 3 * 60; // 3 mins @@ -257,6 +260,11 @@ public void cancelOwnershipMonitor() { } } + @Override + public void cleanOwnerships() { + doCleanup(lookupServiceAddress); + } + public synchronized void start() throws PulsarServerException { if (!validateChannelState(LeaderElectionServiceStarted, false)) { throw new IllegalStateException("Invalid channel state:" + channelState.name()); @@ -284,7 +292,7 @@ public synchronized void start() throws PulsarServerException { } } PulsarClusterMetadataSetup.createNamespaceIfAbsent - (pulsar.getPulsarResources(), NamespaceName.SYSTEM_NAMESPACE, config.getClusterName()); + (pulsar.getPulsarResources(), SYSTEM_NAMESPACE, config.getClusterName()); ExtensibleLoadManagerImpl.createSystemTopic(pulsar, TOPIC); @@ -696,6 +704,8 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { if (isTargetBroker(data.dstBroker())) { log(null, serviceUnit, data, null); lastOwnEventHandledAt = System.currentTimeMillis(); + } else if (data.force() && isTargetBroker(data.sourceBroker())) { + closeServiceUnit(serviceUnit); } } @@ -1110,21 +1120,23 @@ private void scheduleCleanup(String broker, long delayInSecs) { private ServiceUnitStateData getOverrideInactiveBrokerStateData(ServiceUnitStateData orphanData, - String selectedBroker) { + String selectedBroker, + String inactiveBroker) { if (orphanData.state() == Splitting) { return new ServiceUnitStateData(Splitting, orphanData.dstBroker(), selectedBroker, Map.copyOf(orphanData.splitServiceUnitToDestBroker()), true, getNextVersionId(orphanData)); } else { - return new ServiceUnitStateData(Owned, selectedBroker, true, getNextVersionId(orphanData)); + return new ServiceUnitStateData(Owned, selectedBroker, inactiveBroker, + true, getNextVersionId(orphanData)); } } - private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData) { - - Optional selectedBroker = selectBroker(serviceUnit); + private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanData, String inactiveBroker) { + Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker); if (selectedBroker.isPresent()) { - var override = getOverrideInactiveBrokerStateData(orphanData, selectedBroker.get()); + var override = getOverrideInactiveBrokerStateData( + orphanData, selectedBroker.get(), inactiveBroker); log.info("Overriding ownership serviceUnit:{} from orphanData:{} to overrideData:{}", serviceUnit, orphanData, override); publishOverrideEventAsync(serviceUnit, orphanData, override) @@ -1142,26 +1154,69 @@ private void overrideOwnership(String serviceUnit, ServiceUnitStateData orphanDa } } + private void waitForCleanups(String broker, boolean excludeSystemTopics, int maxWaitTimeInMillis) { + long started = System.currentTimeMillis(); + while (System.currentTimeMillis() - started < maxWaitTimeInMillis) { + boolean cleaned = true; + for (var etr : tableview.entrySet()) { + var serviceUnit = etr.getKey(); + var data = etr.getValue(); + + if (excludeSystemTopics && serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { + continue; + } + + if (data.state() == Owned && broker.equals(data.dstBroker())) { + cleaned = false; + break; + } + } + if (cleaned) { + try { + MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_CONVERGENCE_DELAY_IN_MILLIS); + } catch (InterruptedException e) { + log.warn("Interrupted while gracefully waiting for the cleanup convergence."); + } + break; + } else { + try { + MILLISECONDS.sleep(OWNERSHIP_CLEAN_UP_WAIT_RETRY_DELAY_IN_MILLIS); + } catch (InterruptedException e) { + log.warn("Interrupted while delaying the next service unit clean-up. Cleaning broker:{}", + lookupServiceAddress); + } + } + } + } - private void doCleanup(String broker) { + private synchronized void doCleanup(String broker) { long startTime = System.nanoTime(); log.info("Started ownership cleanup for the inactive broker:{}", broker); int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); + Map orphanSystemServiceUnits = new HashMap<>(); for (var etr : tableview.entrySet()) { var stateData = etr.getValue(); var serviceUnit = etr.getKey(); var state = state(stateData); if (StringUtils.equals(broker, stateData.dstBroker())) { if (isActiveState(state)) { - overrideOwnership(serviceUnit, stateData); + if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { + orphanSystemServiceUnits.put(serviceUnit, stateData); + } else { + overrideOwnership(serviceUnit, stateData, broker); + } orphanServiceUnitCleanupCnt++; } } else if (StringUtils.equals(broker, stateData.sourceBroker())) { if (isInFlightState(state)) { - overrideOwnership(serviceUnit, stateData); + if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { + orphanSystemServiceUnits.put(serviceUnit, stateData); + } else { + overrideOwnership(serviceUnit, stateData, broker); + } orphanServiceUnitCleanupCnt++; } } @@ -1170,14 +1225,33 @@ private void doCleanup(String broker) { try { producer.flush(); } catch (PulsarClientException e) { - log.error("Failed to flush the in-flight messages.", e); + log.error("Failed to flush the in-flight non-system bundle override messages.", e); } + if (orphanServiceUnitCleanupCnt > 0) { + // System bundles can contain this channel's system topic and other important system topics. + // Cleaning such system bundles(closing the system topics) together with the non-system bundles + // can cause the cluster to be temporarily unstable. + // Hence, we clean the non-system bundles first and gracefully wait for them. + // After that, we clean the system bundles, if any. + waitForCleanups(broker, true, OWNERSHIP_CLEAN_UP_MAX_WAIT_TIME_IN_MILLIS); this.totalOrphanServiceUnitCleanupCnt += orphanServiceUnitCleanupCnt; this.totalInactiveBrokerCleanupCnt++; } + // clean system bundles in the end + for (var orphanSystemServiceUnit : orphanSystemServiceUnits.entrySet()) { + log.info("Overriding orphan system service unit:{}", orphanSystemServiceUnit.getKey()); + overrideOwnership(orphanSystemServiceUnit.getKey(), orphanSystemServiceUnit.getValue(), broker); + } + + try { + producer.flush(); + } catch (PulsarClientException e) { + log.error("Failed to flush the in-flight system bundle override messages.", e); + } + double cleanupTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - startTime)); @@ -1196,9 +1270,9 @@ private void doCleanup(String broker) { } - private Optional selectBroker(String serviceUnit) { + private Optional selectBroker(String serviceUnit, String inactiveBroker) { try { - return loadManager.selectAsync(getNamespaceBundle(serviceUnit)) + return loadManager.selectAsync(getNamespaceBundle(serviceUnit), Set.of(inactiveBroker)) .get(inFlightStateWaitingTimeInMillis, MILLISECONDS); } catch (Throwable e) { log.error("Failed to select a broker for serviceUnit:{}", serviceUnit); @@ -1206,8 +1280,10 @@ private Optional selectBroker(String serviceUnit) { return Optional.empty(); } - private Optional getRollForwardStateData(String serviceUnit, long nextVersionId) { - Optional selectedBroker = selectBroker(serviceUnit); + private Optional getRollForwardStateData(String serviceUnit, + String inactiveBroker, + long nextVersionId) { + Optional selectedBroker = selectBroker(serviceUnit, inactiveBroker); if (selectedBroker.isEmpty()) { return Optional.empty(); } @@ -1222,7 +1298,7 @@ private Optional getOverrideInFlightStateData( var state = orphanData.state(); switch (state) { case Assigning: { - return getRollForwardStateData(serviceUnit, nextVersionId); + return getRollForwardStateData(serviceUnit, orphanData.dstBroker(), nextVersionId); } case Splitting: { return Optional.of(new ServiceUnitStateData(Splitting, @@ -1235,7 +1311,7 @@ private Optional getOverrideInFlightStateData( // rollback to the src return Optional.of(new ServiceUnitStateData(Owned, orphanData.sourceBroker(), true, nextVersionId)); } else { - return getRollForwardStateData(serviceUnit, nextVersionId); + return getRollForwardStateData(serviceUnit, orphanData.sourceBroker(), nextVersionId); } } default: { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index d72ce9661b988..498f48d16d2cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -882,6 +882,51 @@ SplitDecision.Reason.Unknown, new AtomicLong(6)) assertEquals(actual, expected); } + @Test + public void testDisableBroker() throws Exception { + // Test rollback to modular load manager. + ServiceConfiguration defaultConf = getDefaultConf(); + defaultConf.setAllowAutoTopicCreation(true); + defaultConf.setForceDeleteNamespaceAllowed(true); + defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + defaultConf.setLoadBalancerSheddingEnabled(false); + try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { + var pulsar3 = additionalPulsarTestContext.getPulsarService(); + ExtensibleLoadManagerImpl ternaryLoadManager = spy((ExtensibleLoadManagerImpl) + FieldUtils.readField(pulsar3.getLoadManager().get(), "loadManager", true)); + String topic = "persistent://public/default/test"; + + String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); + TopicName topicName = TopicName.get("test"); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + if (!pulsar3.getBrokerServiceUrl().equals(lookupResult1)) { + admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), + pulsar3.getLookupServiceAddress()); + lookupResult1 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + } + String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic); + String lookupResult3 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + + assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl()); + assertEquals(lookupResult1, lookupResult2); + assertEquals(lookupResult1, lookupResult3); + + + assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + assertTrue(ternaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + + ternaryLoadManager.disableBroker(); + + assertFalse(ternaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + if (primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()) { + assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + } else { + assertTrue(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + } + } + } + private static abstract class MockBrokerFilter implements BrokerFilter { @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 0f682cf048fe9..1263170bec495 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -517,7 +517,7 @@ public void transferTestWhenDestBrokerFails() // recovered, check the monitor update state : Assigned -> Owned doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress1))) - .when(loadManager).selectAsync(any()); + .when(loadManager).selectAsync(any(), any()); FieldUtils.writeDeclaredField(channel2, "producer", producer, true); FieldUtils.writeDeclaredField(channel1, "inFlightStateWaitingTimeInMillis", 1 , true); @@ -715,8 +715,8 @@ public void handleBrokerDeletionEventTest() var cleanupJobs1 = getCleanupJobs(channel1); var cleanupJobs2 = getCleanupJobs(channel2); - var leaderCleanupJobs = spy(cleanupJobs1); - var followerCleanupJobs = spy(cleanupJobs2); + var leaderCleanupJobsTmp = spy(cleanupJobs1); + var followerCleanupJobsTmp = spy(cleanupJobs2); var leaderChannel = channel1; var followerChannel = channel2; String leader = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); @@ -725,10 +725,12 @@ public void handleBrokerDeletionEventTest() if (leader.equals(lookupServiceAddress2)) { leaderChannel = channel2; followerChannel = channel1; - var tmp = followerCleanupJobs; - followerCleanupJobs = leaderCleanupJobs; - leaderCleanupJobs = tmp; + var tmp = followerCleanupJobsTmp; + followerCleanupJobsTmp = leaderCleanupJobsTmp; + leaderCleanupJobsTmp = tmp; } + final var leaderCleanupJobs = leaderCleanupJobsTmp; + final var followerCleanupJobs = followerCleanupJobsTmp; FieldUtils.writeDeclaredField(leaderChannel, "cleanupJobs", leaderCleanupJobs, true); FieldUtils.writeDeclaredField(followerChannel, "cleanupJobs", followerCleanupJobs, @@ -737,7 +739,7 @@ public void handleBrokerDeletionEventTest() var owner1 = channel1.getOwnerAsync(bundle1); var owner2 = channel2.getOwnerAsync(bundle2); doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) - .when(loadManager).selectAsync(any()); + .when(loadManager).selectAsync(any(), any()); assertTrue(owner1.get().isEmpty()); assertTrue(owner2.get().isEmpty()); @@ -771,9 +773,11 @@ public void handleBrokerDeletionEventTest() verify(leaderCleanupJobs, times(1)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(0, leaderCleanupJobs.size()); + assertEquals(0, followerCleanupJobs.size()); + }); - assertEquals(0, leaderCleanupJobs.size()); - assertEquals(0, followerCleanupJobs.size()); validateMonitorCounters(leaderChannel, 1, 0, @@ -799,8 +803,12 @@ public void handleBrokerDeletionEventTest() verify(leaderCleanupJobs, times(2)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); - assertEquals(1, leaderCleanupJobs.size()); - assertEquals(0, followerCleanupJobs.size()); + + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(1, leaderCleanupJobs.size()); + assertEquals(0, followerCleanupJobs.size()); + }); + validateMonitorCounters(leaderChannel, 1, 0, @@ -816,8 +824,12 @@ public void handleBrokerDeletionEventTest() verify(leaderCleanupJobs, times(2)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); - assertEquals(0, leaderCleanupJobs.size()); - assertEquals(0, followerCleanupJobs.size()); + + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(0, leaderCleanupJobs.size()); + assertEquals(0, followerCleanupJobs.size()); + }); + validateMonitorCounters(leaderChannel, 1, 0, @@ -835,8 +847,11 @@ public void handleBrokerDeletionEventTest() verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); - assertEquals(1, leaderCleanupJobs.size()); - assertEquals(0, followerCleanupJobs.size()); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(1, leaderCleanupJobs.size()); + assertEquals(0, followerCleanupJobs.size()); + }); + validateMonitorCounters(leaderChannel, 1, 0, @@ -854,8 +869,11 @@ public void handleBrokerDeletionEventTest() verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); - assertEquals(0, leaderCleanupJobs.size()); - assertEquals(0, followerCleanupJobs.size()); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(0, leaderCleanupJobs.size()); + assertEquals(0, followerCleanupJobs.size()); + }); + validateMonitorCounters(leaderChannel, 2, 0, @@ -880,8 +898,11 @@ public void handleBrokerDeletionEventTest() verify(leaderCleanupJobs, times(3)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); - assertEquals(0, leaderCleanupJobs.size()); - assertEquals(0, followerCleanupJobs.size()); + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(0, leaderCleanupJobs.size()); + assertEquals(0, followerCleanupJobs.size()); + }); + validateMonitorCounters(leaderChannel, 2, 0, @@ -1103,7 +1124,7 @@ public void assignTestWhenDestBrokerProducerFails() FieldUtils.writeDeclaredField(channel2, "inFlightStateWaitingTimeInMillis", 3 * 1000, true); doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) - .when(loadManager).selectAsync(any()); + .when(loadManager).selectAsync(any(), any()); channel1.publishAssignEventAsync(bundle, lookupServiceAddress2); // channel1 is broken. the assign won't be complete. waitUntilState(channel1, bundle); @@ -1442,7 +1463,7 @@ public void testOverrideInactiveBrokerStateData() // test stable metadata state doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) - .when(loadManager).selectAsync(any()); + .when(loadManager).selectAsync(any(), any()); leaderChannel.handleMetadataSessionEvent(SessionReestablished); followerChannel.handleMetadataSessionEvent(SessionReestablished); FieldUtils.writeDeclaredField(leaderChannel, "lastMetadataSessionEventTimestamp", @@ -1507,7 +1528,7 @@ public void testOverrideOrphanStateData() // test stable metadata state doReturn(CompletableFuture.completedFuture(Optional.of(lookupServiceAddress2))) - .when(loadManager).selectAsync(any()); + .when(loadManager).selectAsync(any(), any()); FieldUtils.writeDeclaredField(leaderChannel, "inFlightStateWaitingTimeInMillis", -1, true); FieldUtils.writeDeclaredField(followerChannel, "inFlightStateWaitingTimeInMillis", From 712d80190ab319f3912da8e4647ad952e2ccfd3d Mon Sep 17 00:00:00 2001 From: "Kim, Joo Hyuk" Date: Fri, 19 May 2023 00:21:37 +0900 Subject: [PATCH 118/494] [improve][cli] Allow pulser-client consume create a replicated subscription (#20316) --- .../main/java/org/apache/pulsar/client/cli/CmdConsume.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java index 58ab6360a17bb..0c65604cbe6b8 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/CmdConsume.java @@ -109,6 +109,9 @@ public class CmdConsume extends AbstractCmdConsume { @Parameter(names = { "-pm", "--pool-messages" }, description = "Use the pooled message", arity = 1) private boolean poolMessages = true; + @Parameter(names = {"-rs", "--replicated" }, description = "Whether the subscription status should be replicated") + private boolean replicateSubscriptionState = false; + public CmdConsume() { // Do nothing super(); @@ -156,7 +159,8 @@ private int consume(String topic) { .subscriptionType(subscriptionType) .subscriptionMode(subscriptionMode) .subscriptionInitialPosition(subscriptionInitialPosition) - .poolMessages(poolMessages); + .poolMessages(poolMessages) + .replicateSubscriptionState(replicateSubscriptionState); if (isRegex) { builder.topicsPattern(Pattern.compile(topic)); From ae73fac437e06e7709ff0f443066bff1bb0de220 Mon Sep 17 00:00:00 2001 From: tison Date: Thu, 18 May 2023 23:19:10 +0800 Subject: [PATCH 119/494] [improve][admin] Return BAD_REQUEST on cluster data is null for createCluster (#20346) --- .../java/org/apache/pulsar/broker/admin/impl/ClustersBase.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java index e61a7f20d1d67..5d4ed54c33466 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/ClustersBase.java @@ -162,6 +162,9 @@ public void createCluster( .thenCompose(__ -> validatePoliciesReadOnlyAccessAsync()) .thenCompose(__ -> { NamedEntity.checkName(cluster); + if (clusterData == null) { + throw new RestException(Status.BAD_REQUEST, "cluster data is required"); + } try { clusterData.checkPropertiesIfPresent(); } catch (IllegalArgumentException ex) { From a0609e876ffaf3556c821726af0cd229b8aa2413 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 29 May 2023 18:01:24 +0800 Subject: [PATCH 120/494] [fix][test] Fix SegmentAbortedTxnProcessorTest (#20358) --- .../SegmentAbortedTxnProcessorTest.java | 35 ++++++++++++++++--- .../transaction/TransactionTestBase.java | 6 ++-- .../buffer/TransactionBufferCloseTest.java | 4 --- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java index cb15ab003f7b7..0600833b1adb6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/SegmentAbortedTxnProcessorTest.java @@ -25,6 +25,7 @@ import java.lang.reflect.Field; import java.util.LinkedList; import java.util.NavigableMap; +import java.util.Optional; import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -38,10 +39,12 @@ import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.service.SystemTopicTxnBufferSnapshotService.ReferenceCountedWriter; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.broker.transaction.buffer.AbortedTxnProcessor; +import org.apache.pulsar.broker.transaction.buffer.impl.SingleSnapshotAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.impl.SnapshotSegmentAbortedTxnProcessorImpl; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotIndexes; import org.apache.pulsar.broker.transaction.buffer.metadata.v2.TransactionBufferSnapshotSegment; @@ -56,6 +59,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicStats; import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testcontainers.shaded.org.awaitility.reflect.WhiteboxImpl; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -71,11 +75,13 @@ public class SegmentAbortedTxnProcessorTest extends TransactionTestBase { @Override @BeforeClass protected void setup() throws Exception { - setUpBase(1, 1, PROCESSOR_TOPIC, 0); + setUpBase(1, 1, null, 0); this.pulsarService = getPulsarServiceList().get(0); this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); this.pulsarService.getConfig().setTransactionBufferSnapshotSegmentSize(8 + PROCESSOR_TOPIC.length() + SEGMENT_SIZE * 3); + admin.topics().createNonPartitionedTopic(PROCESSOR_TOPIC); + assertTrue(getSnapshotAbortedTxnProcessor(PROCESSOR_TOPIC) instanceof SnapshotSegmentAbortedTxnProcessorImpl); } @Override @@ -311,15 +317,18 @@ private void doCompaction(TopicName topic) throws Exception { */ @Test public void testSnapshotProcessorUpgrade() throws Exception { + String NAMESPACE2 = TENANT + "/ns2"; + admin.namespaces().createNamespace(NAMESPACE2); this.pulsarService = getPulsarServiceList().get(0); this.pulsarService.getConfig().setTransactionBufferSegmentedSnapshotEnabled(false); // Create a topic, send 10 messages without using transactions, and send 10 messages using transactions. // Abort these transactions and verify the data. - final String topicName = "persistent://" + NAMESPACE1 + "/testSnapshotProcessorUpgrade"; + final String topicName = "persistent://" + NAMESPACE2 + "/testSnapshotProcessorUpgrade"; Producer producer = pulsarClient.newProducer().topic(topicName).create(); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("test-sub").subscribe(); + assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SingleSnapshotAbortedTxnProcessorImpl); // Send 10 messages without using transactions for (int i = 0; i < 10; i++) { producer.send(("test-message-" + i).getBytes()); @@ -352,6 +361,7 @@ public void testSnapshotProcessorUpgrade() throws Exception { // Unload the topic admin.topics().unload(topicName); + assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SnapshotSegmentAbortedTxnProcessorImpl); // Sends a new message using a transaction and aborts it. Transaction txn = pulsarClient.newTransaction() @@ -362,7 +372,7 @@ public void testSnapshotProcessorUpgrade() throws Exception { // Verifies that the topic has exactly one segment. Awaitility.await().untilAsserted(() -> { - String segmentTopic = "persistent://" + NAMESPACE1 + "/" + + String segmentTopic = "persistent://" + NAMESPACE2 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_SEGMENTS; TopicStats topicStats = admin.topics().getStats(segmentTopic); assertEquals(1, topicStats.getMsgInCounter()); @@ -401,7 +411,7 @@ public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Except String topicName = "persistent://" + namespaceName + "/newTopic"; Producer producer = pulsarClient.newProducer().topic(topicName).create(); producer.close(); - + assertTrue(getSnapshotAbortedTxnProcessor(topicName) instanceof SnapshotSegmentAbortedTxnProcessorImpl); // Check that the __transaction_buffer_snapshot topic is not created in the same namespace String transactionBufferSnapshotTopic = "persistent://" + namespaceName + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT; @@ -415,4 +425,21 @@ public void testSegmentedSnapshotWithoutCreatingOldSnapshotTopic() throws Except // Destroy the namespace after the test admin.namespaces().deleteNamespace(namespaceName, true); } + + private AbortedTxnProcessor getSnapshotAbortedTxnProcessor(String topicName) { + PersistentTopic persistentTopic = getPersistentTopic(topicName); + return WhiteboxImpl.getInternalState(persistentTopic.getTransactionBuffer(), "snapshotAbortedTxnProcessor"); + } + + private PersistentTopic getPersistentTopic(String topicName) { + for (PulsarService pulsar : getPulsarServiceList()) { + CompletableFuture> future = + pulsar.getBrokerService().getTopic(topicName, false); + if (future == null) { + continue; + } + return (PersistentTopic) future.join().get(); + } + throw new NullPointerException("topic[" + topicName + "] not found"); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index f45eda8d21fbe..cd0c089ad41be 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -114,10 +114,10 @@ protected void setUpBase(int numBroker,int numPartitionsOfTC, String topic, int new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); createTransactionCoordinatorAssign(numPartitionsOfTC); + admin.tenants().createTenant(TENANT, + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NAMESPACE1); if (topic != null) { - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); - admin.namespaces().createNamespace(NAMESPACE1); if (numPartitions == 0) { admin.topics().createNonPartitionedTopic(topic); } else { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java index e92cf29521e34..d1784f6a392bf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferCloseTest.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.broker.transaction.buffer; -import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -30,7 +29,6 @@ import org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -49,8 +47,6 @@ protected void setup() throws Exception { setUpBase(1, 16, null, 0); Awaitility.await().until(() -> ((PulsarClientImpl) pulsarClient) .getTcClient().getState() == TransactionCoordinatorClient.State.READY); - admin.tenants().createTenant(TENANT, - new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); } @AfterMethod(alwaysRun = true) From af9df45fbb078e3568444bc03bfd01a4be07ed4c Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 24 May 2023 15:06:34 +0800 Subject: [PATCH 121/494] [improve][monitor] Add JVM start time metric (#20381) --- .../main/java/org/apache/pulsar/common/stats/JvmMetrics.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java index 1f15beb8a0b92..8a8da0bb1ac93 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/stats/JvmMetrics.java @@ -99,6 +99,9 @@ public List generate() { Runtime r = Runtime.getRuntime(); + RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); + + m.put("jvm_start_time", runtimeMXBean.getStartTime()); m.put("jvm_heap_used", r.totalMemory() - r.freeMemory()); m.put("jvm_max_memory", r.maxMemory()); m.put("jvm_total_memory", r.totalMemory()); From 0126b0a1f3597ddbb37afc0da9f35701f726dc71 Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Mon, 29 May 2023 22:11:28 +0800 Subject: [PATCH 122/494] [improve] [broker] Avoid `PersistentSubscription.expireMessages` logic check backlog twice. (#20416) --- .../broker/service/persistent/PersistentSubscription.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index d283cac77c7b6..090e8e253776d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -1062,8 +1062,9 @@ public List getConsumers() { @Override public boolean expireMessages(int messageTTLInSeconds) { - if ((getNumberOfEntriesInBacklog(false) == 0) || (dispatcher != null && dispatcher.isConsumerConnected() - && getNumberOfEntriesInBacklog(false) < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK + long backlog = getNumberOfEntriesInBacklog(false); + if (backlog == 0 || (dispatcher != null && dispatcher.isConsumerConnected() + && backlog < MINIMUM_BACKLOG_FOR_EXPIRY_CHECK && !topic.isOldestMessageExpired(cursor, messageTTLInSeconds))) { // don't do anything for almost caught-up connected subscriptions return false; From 0a7ba839c0c54056050435f226bd51f25cda3ac4 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 31 May 2023 10:18:03 +0800 Subject: [PATCH 123/494] [fix][fn] Fix function update error (#19895) --- .../worker/rest/api/FunctionsImpl.java | 5 +++- .../functions/worker/rest/api/SinksImpl.java | 5 +++- .../worker/rest/api/SourcesImpl.java | 5 +++- .../functions/PulsarFunctionsTest.java | 29 +++++++++++++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index c7967600da5f6..715d660ddff0d 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -55,6 +55,7 @@ import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.FunctionsManager; @@ -374,8 +375,10 @@ public void updateFunction(final String tenant, Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder; if (isNotBlank(functionPkgUrl) || uploadedInputStream != null) { + Function.FunctionMetaData metaData = functionMetaDataBuilder.build(); + metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData); try { - packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(), + packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData, functionPkgUrl, fileDetail, componentPackageFile); } catch (Exception e) { log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType), diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 98450c4a0b5d0..5370fe93a7a30 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -54,6 +54,7 @@ import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; +import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; import org.apache.pulsar.functions.utils.SinkConfigUtils; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; @@ -378,8 +379,10 @@ public void updateSink(final String tenant, Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder; if (isNotBlank(sinkPkgUrl) || uploadedInputStream != null) { + Function.FunctionMetaData metaData = functionMetaDataBuilder.build(); + metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData); try { - packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(), + packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData, sinkPkgUrl, fileDetail, componentPackageFile); } catch (Exception e) { log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType), diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index 876c7e7572e78..2f491424d658a 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -53,6 +53,7 @@ import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; +import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; import org.apache.pulsar.functions.utils.SourceConfigUtils; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; @@ -372,8 +373,10 @@ public void updateSource(final String tenant, Function.PackageLocationMetaData.Builder packageLocationMetaDataBuilder; if (isNotBlank(sourcePkgUrl) || uploadedInputStream != null) { + Function.FunctionMetaData metaData = functionMetaDataBuilder.build(); + metaData = FunctionMetaDataUtils.incrMetadataVersion(metaData, metaData); try { - packageLocationMetaDataBuilder = getFunctionPackageLocation(functionMetaDataBuilder.build(), + packageLocationMetaDataBuilder = getFunctionPackageLocation(metaData, sourcePkgUrl, fileDetail, componentPackageFile); } catch (Exception e) { log.error("Failed process {} {}/{}/{} package: ", ComponentTypeUtils.toString(componentType), diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java index 1e54764ad5d2b..e6ba6acff83c0 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/functions/PulsarFunctionsTest.java @@ -729,6 +729,19 @@ protected void testExclamationFunction(Runtime runtime, //get function status getFunctionStatus(functionName, 0, true, 2); + // update code file + switch (runtime) { + case JAVA: + updateFunctionCodeFile(functionName, Runtime.JAVA, "test"); + break; + case PYTHON: + updateFunctionCodeFile(functionName, Runtime.PYTHON, EXCLAMATION_PYTHON_FILE); + break; + case GO: + updateFunctionCodeFile(functionName, Runtime.GO, EXCLAMATION_GO_FILE); + break; + } + // delete function deleteFunction(functionName); @@ -894,6 +907,22 @@ private void updateFunctionParallelism(String functionName, int parallelism) thr assertTrue(result.getStdout().contains("Updated successfully")); } + private void updateFunctionCodeFile(String functionName, Runtime runtime, String codeFile) throws Exception { + + CommandGenerator generator = new CommandGenerator(); + generator.setFunctionName(functionName); + generator.setRuntime(runtime); + String command = generator.generateUpdateFunctionCommand(codeFile); + + log.info("---------- Function command: {}", command); + String[] commands = { + "sh", "-c", command + }; + ContainerExecResult result = pulsarCluster.getAnyWorker().execCmd( + commands); + assertTrue(result.getStdout().contains("Updated successfully")); + } + protected void submitFunction(Runtime runtime, String inputTopicName, String outputTopicName, From eac56db862ea14609f7324e60fbc5f94a550eb8d Mon Sep 17 00:00:00 2001 From: Andy Walker Date: Tue, 30 May 2023 23:03:36 -0400 Subject: [PATCH 124/494] [fix][fn] Go functions must retrieve consumers by non-particioned topic ID (#20413) Co-authored-by: Andy Walker --- pulsar-function-go/pf/instance.go | 18 ++++++++++++++++-- pulsar-function-go/pf/util.go | 11 +++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index 138489444d160..6f73f3e631212 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -404,11 +404,25 @@ func (gi *goInstance) processResult(msgInput pulsar.Message, output []byte) { // ackInputMessage doesn't produce any result, or the user doesn't want the result. func (gi *goInstance) ackInputMessage(inputMessage pulsar.Message) { log.Debugf("ack input message topic name is: %s", inputMessage.Topic()) - gi.consumers[inputMessage.Topic()].Ack(inputMessage) + gi.respondMessage(inputMessage, true) } func (gi *goInstance) nackInputMessage(inputMessage pulsar.Message) { - gi.consumers[inputMessage.Topic()].Nack(inputMessage) + gi.respondMessage(inputMessage, false) +} + +func (gi *goInstance) respondMessage(inputMessage pulsar.Message, ack bool) { + topicName, err := ParseTopicName(inputMessage.Topic()) + if err != nil { + log.Errorf("unable respond to message ID %s - invalid topic: %v", messageIDStr(inputMessage), err) + return + } + // consumers are indexed by topic name only (no partition) + if ack { + gi.consumers[topicName.NameWithoutPartition()].Ack(inputMessage) + return + } + gi.consumers[topicName.NameWithoutPartition()].Nack(inputMessage) } func getIdleTimeout(timeoutMilliSecond time.Duration) time.Duration { diff --git a/pulsar-function-go/pf/util.go b/pulsar-function-go/pf/util.go index d5b32da841121..1d1aa1cab939f 100644 --- a/pulsar-function-go/pf/util.go +++ b/pulsar-function-go/pf/util.go @@ -21,6 +21,8 @@ package pf import ( "fmt" + + "github.com/apache/pulsar-client-go/pulsar" ) func getProperties(fullyQualifiedName string, instanceID int) map[string]string { @@ -39,3 +41,12 @@ func getDefaultSubscriptionName(tenant, namespace, name string) string { func getFullyQualifiedInstanceID(tenant, namespace, name string, instanceID int) string { return fmt.Sprintf("%s/%s/%s:%d", tenant, namespace, name, instanceID) } + +func messageIDStr(msg pulsar.Message) string { + // ::: + return fmt.Sprintf("%d:%d:%d:%d", + msg.ID().LedgerID(), + msg.ID().EntryID(), + msg.ID().PartitionIdx(), + msg.ID().BatchIdx()) +} From c776e893574e3a0ed53bde1e22df959d0f12c47e Mon Sep 17 00:00:00 2001 From: Andy Walker Date: Thu, 1 Jun 2023 13:42:04 -0400 Subject: [PATCH 125/494] [fix][fn]Reset idle timer correctly (#20450) Co-authored-by: Andy Walker Fix apache/pulsar#20449 Master Issue: #20449 ### Verifying this change - [ ] Make sure that the change passes the CI checks. This change is a trivial rework / code cleanup without any test coverage. ### Does this pull request potentially affect one of the following parts: *If the box was checked, please highlight the changes* - [ ] Dependencies (add or upgrade a dependency) - [ ] The public API - [ ] The schema - [ ] The default values of configurations - [ ] The threading model - [ ] The binary protocol - [ ] The REST endpoints - [ ] The admin CLI options - [ ] The metrics - [ ] Anything that affects deployment ### Documentation - [ ] `doc` - [ ] `doc-required` - [X] `doc-not-needed` - [ ] `doc-complete` ### Matching PR in forked repository PR in forked repository: https://github.com/flowchartsman/pulsar/pull/5 (cherry picked from commit e05b890b804cee781e87db750f2afc3a382fa1d2) --- pulsar-function-go/pf/instance.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index 6f73f3e631212..2c2deb19b2813 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -149,7 +149,6 @@ func (gi *goInstance) startFunction(function function) error { defer metricsServicer.close() CLOSE: for { - idleTimer.Reset(idleDuration) select { case cm := <-channel: msgInput := cm.Message @@ -181,6 +180,11 @@ CLOSE: close(channel) break CLOSE } + // reset the idle timer and drain if appropriate before the next loop + if !idleTimer.Stop() { + <-idleTimer.C + } + idleTimer.Reset(idleDuration) } gi.closeLogTopic() From a125e159c2db7c673dea795bc061dd6bfaeabd52 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Mon, 29 May 2023 21:03:34 +0800 Subject: [PATCH 126/494] [fix][io] Close the kafka source connector if there is uncaught exception (#20424) Fixes #19880 ### Motivation If any unexpected and uncaught exceptions are thrown in the `runnerThread` of the Kafka source connector, the connector will just log that message and get stuck. The connector won't exit but won't do anything either. ### Modifications * Close the connector if there is uncaught exception Signed-off-by: Zike Yang (cherry picked from commit 6079a9186ac8d1c96c97828a289d7865c5890ebd) --- .../pulsar/io/kafka/KafkaAbstractSource.java | 9 +++++++- .../kafka/source/KafkaAbstractSourceTest.java | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java index 565c36047474b..8d2cbd8e74e14 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java @@ -189,7 +189,14 @@ public void start() { } }); runnerThread.setUncaughtExceptionHandler( - (t, e) -> LOG.error("[{}] Error while consuming records", t.getName(), e)); + (t, e) -> { + LOG.error("[{}] Error while consuming records", t.getName(), e); + try { + this.close(); + } catch (InterruptedException ex) { + // The interrupted exception is thrown by the runnerThread itself. Ignore it. + } + }); runnerThread.setName("Kafka Source Thread"); runnerThread.start(); } diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java index 612cf0bc6d2b1..bc06c3e1935b4 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java @@ -20,7 +20,10 @@ import com.google.common.collect.ImmutableMap; +import java.util.Collection; import java.util.Collections; +import java.lang.reflect.Field; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -28,6 +31,7 @@ import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.kafka.KafkaAbstractSource; import org.apache.pulsar.io.kafka.KafkaSourceConfig; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -153,6 +157,25 @@ public final void loadFromSaslYamlFileTest() throws IOException { assertEquals(config.getSslTruststorePassword(), "cert_pwd"); } + @Test + public final void closeConnectorWhenUnexpectedExceptionThrownTest() throws Exception { + KafkaAbstractSource source = new DummySource(); + Consumer consumer = mock(Consumer.class); + Mockito.doThrow(new RuntimeException("Uncaught exception")).when(consumer) + .subscribe(Mockito.any(Collection.class)); + + Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer"); + consumerField.setAccessible(true); + consumerField.set(source, consumer); + + source.start(); + + Field runningField = KafkaAbstractSource.class.getDeclaredField("running"); + runningField.setAccessible(true); + + Assert.assertFalse((boolean) runningField.get(source)); + } + private File getFile(String name) { ClassLoader classLoader = getClass().getClassLoader(); return new File(classLoader.getResource(name).getFile()); From 212a4828ced2d4ab6d8edf8219ec60c82810a6d2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 1 Jun 2023 22:19:49 +0300 Subject: [PATCH 127/494] [fix][sec] Upgrade Guava to 32.0.0 to address CVE-2023-2976 (#20459) (cherry picked from commit 57f9467a8dbcd546ee9127d8dfbd000b46333f23) # Conflicts: # pom.xml # pulsar-sql/presto-distribution/LICENSE --- buildtools/pom.xml | 2 +- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- distribution/shell/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 4 ++-- pulsar-sql/presto-distribution/pom.xml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 62d805b65fa2b..9514a0fc4ee78 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -49,7 +49,7 @@ 3.1.2 4.1.93.Final 4.2.3 - 31.0.1-jre + 32.0.0-jre 1.10.12 2.0 3.12.4 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index a0db421c624e1..b3a573ebc269b 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -265,7 +265,7 @@ The Apache Software License, Version 2.0 - com.google.code.gson-gson-2.8.9.jar - io.gsonfire-gson-fire-1.8.5.jar * Guava - - com.google.guava-guava-31.0.1-jre.jar + - com.google.guava-guava-32.0.0-jre.jar - com.google.guava-failureaccess-1.0.1.jar - com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- com.google.j2objc-j2objc-annotations-1.3.jar @@ -516,7 +516,7 @@ MIT License - org.slf4j-slf4j-api-1.7.32.jar - org.slf4j-jcl-over-slf4j-1.7.32.jar * The Checker Framework - - org.checkerframework-checker-qual-3.12.0.jar + - org.checkerframework-checker-qual-3.33.0.jar * oshi - com.github.oshi-oshi-core-java11-6.4.0.jar * Auth0, Inc. diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 3ccbd9316339f..3040091789cc0 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -326,7 +326,7 @@ The Apache Software License, Version 2.0 * Gson - gson-2.8.9.jar * Guava - - guava-31.0.1-jre.jar + - guava-32.0.0-jre.jar - failureaccess-1.0.1.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- j2objc-annotations-1.3.jar @@ -422,7 +422,7 @@ MIT License * SLF4J -- ../licenses/LICENSE-SLF4J.txt - slf4j-api-1.7.32.jar * The Checker Framework - - checker-qual-3.12.0.jar + - checker-qual-3.33.0.jar Protocol Buffers License * Protocol Buffers diff --git a/pom.xml b/pom.xml index b522ea935b001..47fa6705f1574 100644 --- a/pom.xml +++ b/pom.xml @@ -201,7 +201,7 @@ flexible messaging model and an intuitive client API. 2.10.2 3.3.4 2.4.15 - 31.0.1-jre + 32.0.0-jre 1.0 0.16.1 6.2.8 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index e737fbf6ee2e6..7f78a9c06f8a5 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -221,7 +221,7 @@ The Apache Software License, Version 2.0 - jackson-module-jaxb-annotations-2.14.2.jar - jackson-module-jsonSchema-2.14.2.jar * Guava - - guava-31.0.1-jre.jar + - guava-32.0.0-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - failureaccess-1.0.1.jar * Google Guice @@ -519,7 +519,7 @@ MIT License * JCL 1.2 Implemented Over SLF4J - jcl-over-slf4j-1.7.32.jar * Checker Qual - - checker-qual-3.12.0.jar + - checker-qual-3.33.0.jar * Annotations - animal-sniffer-annotations-1.19.jar - annotations-4.1.1.4.jar diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 940d38cc9b0b8..c3727f14eed16 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -37,7 +37,7 @@ 2.6 0.0.12 3.0.5 - 31.0.1-jre + 32.0.0-jre 2.12.1 2.5.1 4.0.1 From e122f293d0feddc715a0fb0a3f07152e3c33adf8 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 1 Jun 2023 23:07:11 +0300 Subject: [PATCH 128/494] [fix][test] Remove dependency on httpbin.org service in FunctionCommonTest (#20464) (cherry picked from commit a5f2d25e0f435fa8758904a0f71ceefceb8b5deb) --- pulsar-functions/utils/pom.xml | 9 ++++++++- .../functions/utils/FunctionCommonTest.java | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index 305cd523e2210..f9d4327c070ea 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -99,6 +99,13 @@ ${project.version} + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + @@ -119,7 +126,7 @@ - + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java index 113824fc7c1a1..131f153b08d68 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java @@ -18,12 +18,19 @@ */ package org.apache.pulsar.functions.utils; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.configureFor; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import com.github.tomakehurst.wiremock.WireMockServer; import java.io.File; import java.util.Collection; +import lombok.Cleanup; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.functions.api.Context; @@ -87,7 +94,14 @@ public void testDownloadFile() throws Exception { @Test public void testDownloadFileWithBasicAuth() throws Exception { - final String jarHttpUrl = "https://foo:bar@httpbin.org/basic-auth/foo/bar"; + @Cleanup("stop") + WireMockServer server = new WireMockServer(0); + server.start(); + configureFor(server.port()); + stubFor(get(urlPathEqualTo("/")) + .withBasicAuth("foo", "bar") + .willReturn(aResponse().withBody("Hello world!").withStatus(200))); + final String jarHttpUrl = "http://foo:bar@localhost:" + server.port() + "/"; final File file = Files.newTemporaryFile(); file.deleteOnExit(); assertThat(file.length()).isZero(); From 92bd56d329f74cf399d344d3aebf9d729b5f72c4 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 1 Jun 2023 21:37:32 -0500 Subject: [PATCH 129/494] [fix][test] Replace test call to Auth0 with call to WireMock (#20465) (cherry picked from commit 174c4866f926b1ae7f7935395fbea8c0aeb64e31) --- pulsar-broker/pom.xml | 7 ++ ...uth2AuthenticatedProducerConsumerTest.java | 119 ++++++++++++++++-- 2 files changed, 115 insertions(+), 11 deletions(-) diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 25b31b4f2b7bd..4316ace051a07 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -151,6 +151,13 @@ test + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + io.dropwizard.metrics diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java index 22834b2e0c9c1..cf85ddd913b8b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java @@ -18,15 +18,34 @@ */ package org.apache.pulsar.client.api; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.FileSource; +import com.github.tomakehurst.wiremock.extension.Parameters; +import com.github.tomakehurst.wiremock.extension.ResponseTransformer; +import com.github.tomakehurst.wiremock.http.Request; +import com.github.tomakehurst.wiremock.http.Response; import com.google.common.collect.Sets; +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.impl.DefaultJwtBuilder; +import io.jsonwebtoken.security.Keys; import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.KeyPair; +import java.security.PrivateKey; import java.time.Duration; +import java.util.Base64; +import java.util.Date; import java.util.HashSet; import java.util.Properties; import java.util.Set; @@ -55,22 +74,51 @@ public class TokenOauth2AuthenticatedProducerConsumerTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(TokenOauth2AuthenticatedProducerConsumerTest.class); - // public key in oauth2 server to verify the client passed in token. get from https://jwt.io/ - private final String TOKEN_TEST_PUBLIC_KEY = "data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRmWjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDHy5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB"; + private WireMockServer server; private final String ADMIN_ROLE = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients"; // Credentials File, which contains "client_id" and "client_secret" private final String CREDENTIALS_FILE = "./src/test/resources/authentication/token/credentials_file.json"; - private final String ISSUER_URL = "https://dev-kt-aa9ne.us.auth0.com"; private final String AUDIENCE = "https://dev-kt-aa9ne.us.auth0.com/api/v2/"; @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { + // Create the token key pair + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + + // Start mocked OAuth2 server + server = new WireMockServer(wireMockConfig().port(0).extensions(new OAuth2Transformer(keyPair, 3000))); + server.start(); + + // Set up a correct openid-configuration that points to the next stub + server.stubFor( + get(urlEqualTo("/.well-known/openid-configuration")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "%s", + "token_endpoint": "%s/oauth/token" + } + """.replace("%s", server.baseUrl())))); + + // Only respond when the client sends the expected request body + server.stubFor( + post(urlEqualTo("/oauth/token")) + .withRequestBody( + equalTo("audience=https%3A%2F%2Fdev-kt-aa9ne.us.auth0.com%2Fapi%2Fv2%2F&" + + "client_id=Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x&" + + "client_secret=rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N" + + "_poGAb&grant_type=client_credentials")) + .willReturn(aResponse() + .withTransformers("o-auth-token-transformer") + .withStatus(200))); + conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); - conf.setAuthenticationRefreshCheckSeconds(5); + conf.setAuthenticationRefreshCheckSeconds(1); Set superUserRoles = new HashSet<>(); superUserRoles.add(ADMIN_ROLE); @@ -83,7 +131,7 @@ protected void setup() throws Exception { conf.setBrokerClientAuthenticationPlugin(AuthenticationOAuth2.class.getName()); conf.setBrokerClientAuthenticationParameters("{\n" + " \"privateKey\": \"" + CREDENTIALS_FILE + "\",\n" - + " \"issuerUrl\": \"" + ISSUER_URL + "\",\n" + + " \"issuerUrl\": \"" + server.baseUrl() + "\",\n" + " \"audience\": \"" + AUDIENCE + "\",\n" + "}\n"); @@ -91,7 +139,8 @@ protected void setup() throws Exception { // Set provider domain name Properties properties = new Properties(); - properties.setProperty("tokenPublicKey", TOKEN_TEST_PUBLIC_KEY); + properties.setProperty("tokenPublicKey", "data:;base64," + + Base64.getEncoder().encodeToString(keyPair.getPublic().getEncoded())); conf.setProperties(properties); super.init(); @@ -104,7 +153,7 @@ protected final void clientSetup() throws Exception { // AuthenticationOAuth2 Authentication authentication = AuthenticationFactoryOAuth2.clientCredentials( - new URL(ISSUER_URL), + new URL(server.baseUrl()), path.toUri().toURL(), // key file path AUDIENCE ); @@ -122,6 +171,7 @@ protected final void clientSetup() throws Exception { @Override protected void cleanup() throws Exception { super.internalCleanup(); + server.stop(); } @DataProvider(name = "batch") @@ -210,12 +260,11 @@ public void testOAuth2TokenRefreshedWithoutReconnect() throws Exception { String accessTokenOld = producerImpl.getClientCnx().getAuthenticationDataProvider().getCommandData(); long lastDisconnectTime = producer.getLastDisconnectedTimestamp(); - // the token expire duration is 10 seconds, so we need to wait for the authenticationData refreshed + // the token expire duration is 3 seconds, so we need to wait for the authenticationData refreshed Awaitility.await() - .atLeast(10, TimeUnit.SECONDS) - .atMost(20, TimeUnit.SECONDS) + .atMost(10, TimeUnit.SECONDS) .with() - .pollInterval(Duration.ofSeconds(1)) + .pollInterval(Duration.ofMillis(250)) .untilAsserted(() -> { String accessTokenNew = producerImpl.getClientCnx().getAuthenticationDataProvider().getCommandData(); assertNotEquals(accessTokenNew, accessTokenOld); @@ -243,4 +292,52 @@ public void testOAuth2TokenRefreshedWithoutReconnect() throws Exception { consumer.acknowledgeCumulative(msg); consumer.close(); } + + class OAuth2Transformer extends ResponseTransformer { + + private final PrivateKey privateKey; + private final long tokenTTL; + + OAuth2Transformer(KeyPair key, long tokenTTLMillis) { + this.privateKey = key.getPrivate(); + this.tokenTTL = tokenTTLMillis; + } + + @Override + public Response transform(Request request, Response response, FileSource files, Parameters parameters) { + return Response.Builder.like(response).but().body(""" + { + "access_token": "%s", + "expires_in": %d, + "token_type":"Bearer" + } + """.formatted(generateToken(), + TimeUnit.MILLISECONDS.toSeconds(tokenTTL))).build(); + } + + @Override + public String getName() { + return "o-auth-token-transformer"; + } + + @Override + public boolean applyGlobally() { + return false; + } + + private String generateToken() { + long now = System.currentTimeMillis(); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + defaultJwtBuilder.setHeaderParam("typ", "JWT"); + defaultJwtBuilder.setHeaderParam("alg", "RS256"); + defaultJwtBuilder.setIssuer(server.baseUrl()); + defaultJwtBuilder.setSubject(ADMIN_ROLE); + defaultJwtBuilder.setAudience(AUDIENCE); + defaultJwtBuilder.setIssuedAt(new Date(now)); + defaultJwtBuilder.setNotBefore(new Date(now)); + defaultJwtBuilder.setExpiration(new Date(now + tokenTTL)); + defaultJwtBuilder.signWith(privateKey); + return defaultJwtBuilder.compact(); + } + } } From cd08427cf966aa34e124f156e912e5ff5b050e05 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 2 Jun 2023 18:47:53 -0500 Subject: [PATCH 130/494] [fix][fn] Support customizing TLS config for function download command (#20482) (cherry picked from commit ceed19cf3b8536a8c9059bfbcb29ef972841b412) --- .../runtime/kubernetes/KubernetesRuntime.java | 10 ++++++ .../kubernetes/KubernetesRuntimeTest.java | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 939a446d7fed7..7779422965ba4 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -885,6 +885,16 @@ && isNotBlank(authConfig.getClientAuthenticationParameters())) { "--auth-params", authConfig.getClientAuthenticationParameters())); } + cmd.addAll(Arrays.asList( + "--tls-allow-insecure", + Boolean.toString(authConfig.isTlsAllowInsecureConnection()), + "--tls-enable-hostname-verification", + Boolean.toString(authConfig.isTlsHostnameVerificationEnable()))); + if (isNotBlank(authConfig.getTlsTrustCertsFilePath())) { + cmd.addAll(Arrays.asList( + "--tls-trust-cert-path", + authConfig.getTlsTrustCertsFilePath())); + } } cmd.addAll(Arrays.asList( diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 3facd37fc9244..d6135737c4f21 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -852,6 +852,7 @@ public void testCustomKubernetesDownloadCommandsWithAuth() throws Exception { V1StatefulSet spec = container.createStatefulSet(); String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}" + + " --tls-allow-insecure false --tls-enable-hostname-verification false" + " functions download " + "--tenant " + TEST_TENANT + " --namespace " + TEST_NAMESPACE @@ -878,6 +879,38 @@ public void testCustomKubernetesDownloadCommandsWithAuthWithoutAuthSpec() throws V1StatefulSet spec = container.createStatefulSet(); String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}" + + " --tls-allow-insecure false --tls-enable-hostname-verification false" + + " functions download " + + "--tenant " + TEST_TENANT + + " --namespace " + TEST_NAMESPACE + + " --name " + TEST_NAME + + " --destination-file " + pulsarRootDir + "/" + userJarFile; + String containerCommand = spec.getSpec().getTemplate().getSpec().getContainers().get(0).getCommand().get(2); + assertTrue(containerCommand.contains(expectedDownloadCommand), "Found:" + containerCommand); + } + + @Test + public void testCustomKubernetesDownloadCommandsWithAuthAndCustomTLSWithoutAuthSpec() throws Exception { + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false); + config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false)); + + factory = createKubernetesRuntimeFactory(null, + 10, 1.0, 1.0, Optional.empty(), null, wconfig -> { + wconfig.setAuthenticationEnabled(true); + }, AuthenticationConfig.builder() + .clientAuthenticationPlugin("com.MyAuth") + .clientAuthenticationParameters("{\"authParam1\": \"authParamValue1\"}") + .useTls(true) // set to verify it is ignored because pulsar admin does not consider this setting + .tlsHostnameVerificationEnable(true) + .tlsTrustCertsFilePath("/my/ca.pem") + .build()); + + KubernetesRuntime container = factory.createContainer(config, userJarFile, userJarFile, null, null, 30l); + V1StatefulSet spec = container.createStatefulSet(); + String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl + + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}" + + " --tls-allow-insecure false --tls-enable-hostname-verification true" + + " --tls-trust-cert-path /my/ca.pem" + " functions download " + "--tenant " + TEST_TENANT + " --namespace " + TEST_NAMESPACE From 965cb3829bdcbaed4572c811384c7f9b0b6e7d67 Mon Sep 17 00:00:00 2001 From: Andy Walker Date: Mon, 5 Jun 2023 09:16:51 -0400 Subject: [PATCH 131/494] [fix][fn] Go functions need to use static grpcPort in k8s runtime (#20404) (cherry picked from commit 7e6ca31dcb7bd04afff4daf50ac44be4749b6f2b) --- .../runtime/kubernetes/KubernetesRuntimeFactory.java | 5 +++++ .../functions/runtime/kubernetes/KubernetesRuntimeTest.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java index 3e1d40e80dc6e..bbb6e3992a018 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeFactory.java @@ -302,6 +302,11 @@ public KubernetesRuntime createContainer(InstanceConfig instanceConfig, String c .map((customizer) -> customizer.customizeName(instanceConfig.getFunctionDetails(), jobName)) .orElse(jobName); + // pass grpcPort configured in functionRuntimeFactoryConfigs.grpcPort in functions_worker.yml + if (grpcPort != null) { + instanceConfig.setPort(grpcPort); + } + // pass metricsPort configured in functionRuntimeFactoryConfigs.metricsPort in functions_worker.yml if (metricsPort != null) { instanceConfig.setMetricsPort(metricsPort); diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index d6135737c4f21..2cbbeb8a00b92 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -984,7 +984,7 @@ private void verifyGolangInstance(InstanceConfig config) throws Exception { assertEquals(goInstanceConfig.get("disk"), 10000); assertEquals(goInstanceConfig.get("instanceID"), 0); assertEquals(goInstanceConfig.get("cleanupSubscription"), false); - assertEquals(goInstanceConfig.get("port"), 0); + assertEquals(goInstanceConfig.get("port"), 4332); assertEquals(goInstanceConfig.get("subscriptionType"), 0); assertEquals(goInstanceConfig.get("timeoutMs"), 0); assertEquals(goInstanceConfig.get("subscriptionName"), ""); From 891a62103ae6a1f689143a85501cb80f5df8c554 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 5 Jun 2023 23:00:50 +0300 Subject: [PATCH 132/494] [improve][ci] Increase Maven max heap size to 1024m in all GHA workflows (#20487) (cherry picked from commit 4a89f028a3e33c18e2d9efa4e2537ac9f9e82a08) --- .github/workflows/ci-go-functions.yaml | 2 +- .github/workflows/ci-maven-cache-update.yaml | 2 +- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index 50f003e27315f..e347890a1d1c1 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -33,7 +33,7 @@ concurrency: cancel-in-progress: true env: - MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: preconditions: diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml index 15fefaf3f1645..f20feeea5c9db 100644 --- a/.github/workflows/ci-maven-cache-update.yaml +++ b/.github/workflows/ci-maven-cache-update.yaml @@ -42,7 +42,7 @@ on: - cron: '30 */12 * * *' env: - MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: update-maven-dependencies-cache: diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 06edbae51adde..090221e699d01 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -24,7 +24,7 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xss1500k -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 + MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000 jobs: run-owasp-dependency-check: From b53d3450db2dbee8e6bdd3539b3a86e78e26bd7b Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 6 Jun 2023 00:03:05 -0500 Subject: [PATCH 133/494] [fix][test] Replace calls to Auth0 with calls to wiremock (#20500) (cherry picked from commit da04f24b6e81f05ace04fb4ae0d9a720b44c845f) --- .../broker/auth/MockOIDCIdentityProvider.java | 150 ++++++++++++++++++ ...uth2AuthenticatedProducerConsumerTest.java | 131 +++------------ .../token/credentials_file.json | 4 +- pulsar-proxy/pom.xml | 6 + .../proxy/server/ProxyTlsTestWithAuth.java | 15 +- pulsar-testclient/pom.xml | 7 + .../Oauth2PerformanceTransactionTest.java | 24 ++- .../token/credentials_file.json | 4 +- 8 files changed, 208 insertions(+), 133 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java new file mode 100644 index 0000000000000..5d29c443d2bde --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockOIDCIdentityProvider.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.auth; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.FileSource; +import com.github.tomakehurst.wiremock.extension.Parameters; +import com.github.tomakehurst.wiremock.extension.ResponseTransformer; +import com.github.tomakehurst.wiremock.http.Request; +import com.github.tomakehurst.wiremock.http.Response; +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.impl.DefaultJwtBuilder; +import io.jsonwebtoken.security.Keys; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.util.Base64; +import java.util.Date; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Mock OIDC (and therefore OAuth2) server for testing. Note that the client_id is mapped to the token's subject claim. + */ +public class MockOIDCIdentityProvider { + private final WireMockServer server; + private final PublicKey publicKey; + private final String audience; + public MockOIDCIdentityProvider(String clientSecret, String audience, long tokenTTLMillis) { + this.audience = audience; + KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); + publicKey = keyPair.getPublic(); + server = new WireMockServer(wireMockConfig().port(0) + .extensions(new OAuth2Transformer(keyPair, tokenTTLMillis))); + server.start(); + + // Set up a correct openid-configuration that points to the next stub + server.stubFor( + get(urlEqualTo("/.well-known/openid-configuration")) + .willReturn(aResponse() + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "issuer": "%s", + "token_endpoint": "%s/oauth/token" + } + """.replace("%s", server.baseUrl())))); + + // Only respond when the client sends the expected request body + server.stubFor(post(urlEqualTo("/oauth/token")) + .withRequestBody(matching(".*grant_type=client_credentials.*")) + .withRequestBody(matching(".*audience=" + URLEncoder.encode(audience, StandardCharsets.UTF_8) + ".*")) + .withRequestBody(matching(".*client_id=.*")) + .withRequestBody(matching(".*client_secret=" + clientSecret + "(&.*|$)")) + .willReturn(aResponse().withTransformers("o-auth-token-transformer").withStatus(200))); + } + + public void stop() { + server.stop(); + } + + public String getBase64EncodedPublicKey() { + return Base64.getEncoder().encodeToString(publicKey.getEncoded()); + } + + public String getIssuer() { + return server.baseUrl(); + } + + class OAuth2Transformer extends ResponseTransformer { + + private final PrivateKey privateKey; + private final long tokenTTL; + + private final Pattern clientIdToRolePattern = Pattern.compile("client_id=([A-Za-z0-9-]*)(&|$)"); + + OAuth2Transformer(KeyPair key, long tokenTTLMillis) { + this.privateKey = key.getPrivate(); + this.tokenTTL = tokenTTLMillis; + } + + @Override + public Response transform(Request request, Response response, FileSource files, Parameters parameters) { + Matcher m = clientIdToRolePattern.matcher(request.getBodyAsString()); + if (m.find()) { + String role = m.group(1); + return Response.Builder.like(response).but().body(""" + { + "access_token": "%s", + "expires_in": %d, + "token_type":"Bearer" + } + """.formatted(generateToken(role), + TimeUnit.MILLISECONDS.toSeconds(tokenTTL))).build(); + } else { + return Response.Builder.like(response).but().body("Invalid request").status(400).build(); + } + } + + @Override + public String getName() { + return "o-auth-token-transformer"; + } + + @Override + public boolean applyGlobally() { + return false; + } + + private String generateToken(String role) { + long now = System.currentTimeMillis(); + DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); + defaultJwtBuilder.setHeaderParam("typ", "JWT"); + defaultJwtBuilder.setHeaderParam("alg", "RS256"); + defaultJwtBuilder.setIssuer(server.baseUrl()); + defaultJwtBuilder.setSubject(role); + defaultJwtBuilder.setAudience(audience); + defaultJwtBuilder.setIssuedAt(new Date(now)); + defaultJwtBuilder.setNotBefore(new Date(now)); + defaultJwtBuilder.setExpiration(new Date(now + tokenTTL)); + defaultJwtBuilder.signWith(privateKey); + return defaultJwtBuilder.compact(); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java index cf85ddd913b8b..fdf41c4a6ada1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java @@ -18,38 +18,20 @@ */ package org.apache.pulsar.client.api; -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; -import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static com.github.tomakehurst.wiremock.client.WireMock.post; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; -import com.github.tomakehurst.wiremock.WireMockServer; -import com.github.tomakehurst.wiremock.common.FileSource; -import com.github.tomakehurst.wiremock.extension.Parameters; -import com.github.tomakehurst.wiremock.extension.ResponseTransformer; -import com.github.tomakehurst.wiremock.http.Request; -import com.github.tomakehurst.wiremock.http.Response; import com.google.common.collect.Sets; -import io.jsonwebtoken.SignatureAlgorithm; -import io.jsonwebtoken.impl.DefaultJwtBuilder; -import io.jsonwebtoken.security.Keys; import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.KeyPair; -import java.security.PrivateKey; import java.time.Duration; -import java.util.Base64; -import java.util.Date; import java.util.HashSet; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.impl.ProducerImpl; @@ -60,7 +42,9 @@ import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -74,54 +58,28 @@ public class TokenOauth2AuthenticatedProducerConsumerTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(TokenOauth2AuthenticatedProducerConsumerTest.class); - private WireMockServer server; - - private final String ADMIN_ROLE = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients"; + private MockOIDCIdentityProvider server; // Credentials File, which contains "client_id" and "client_secret" private final String CREDENTIALS_FILE = "./src/test/resources/authentication/token/credentials_file.json"; - private final String AUDIENCE = "https://dev-kt-aa9ne.us.auth0.com/api/v2/"; + private final String audience = "my-pulsar-cluster"; + + @BeforeClass(alwaysRun = true) + protected void setupClass() { + String clientSecret = "super-secret-client-secret"; + server = new MockOIDCIdentityProvider(clientSecret, audience, 3000); + } @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { - // Create the token key pair - KeyPair keyPair = Keys.keyPairFor(SignatureAlgorithm.RS256); - - // Start mocked OAuth2 server - server = new WireMockServer(wireMockConfig().port(0).extensions(new OAuth2Transformer(keyPair, 3000))); - server.start(); - - // Set up a correct openid-configuration that points to the next stub - server.stubFor( - get(urlEqualTo("/.well-known/openid-configuration")) - .willReturn(aResponse() - .withHeader("Content-Type", "application/json") - .withBody(""" - { - "issuer": "%s", - "token_endpoint": "%s/oauth/token" - } - """.replace("%s", server.baseUrl())))); - - // Only respond when the client sends the expected request body - server.stubFor( - post(urlEqualTo("/oauth/token")) - .withRequestBody( - equalTo("audience=https%3A%2F%2Fdev-kt-aa9ne.us.auth0.com%2Fapi%2Fv2%2F&" - + "client_id=Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x&" - + "client_secret=rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N" - + "_poGAb&grant_type=client_credentials")) - .willReturn(aResponse() - .withTransformers("o-auth-token-transformer") - .withStatus(200))); - conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setAuthenticationRefreshCheckSeconds(1); Set superUserRoles = new HashSet<>(); - superUserRoles.add(ADMIN_ROLE); + // Matches the role in th credentials file + superUserRoles.add("my-admin-role"); conf.setSuperUserRoles(superUserRoles); Set providers = new HashSet<>(); @@ -131,16 +89,15 @@ protected void setup() throws Exception { conf.setBrokerClientAuthenticationPlugin(AuthenticationOAuth2.class.getName()); conf.setBrokerClientAuthenticationParameters("{\n" + " \"privateKey\": \"" + CREDENTIALS_FILE + "\",\n" - + " \"issuerUrl\": \"" + server.baseUrl() + "\",\n" - + " \"audience\": \"" + AUDIENCE + "\",\n" + + " \"issuerUrl\": \"" + server.getIssuer() + "\",\n" + + " \"audience\": \"" + audience + "\",\n" + "}\n"); conf.setClusterName("test"); // Set provider domain name Properties properties = new Properties(); - properties.setProperty("tokenPublicKey", "data:;base64," - + Base64.getEncoder().encodeToString(keyPair.getPublic().getEncoded())); + properties.setProperty("tokenPublicKey", "data:;base64," + server.getBase64EncodedPublicKey()); conf.setProperties(properties); super.init(); @@ -153,9 +110,9 @@ protected final void clientSetup() throws Exception { // AuthenticationOAuth2 Authentication authentication = AuthenticationFactoryOAuth2.clientCredentials( - new URL(server.baseUrl()), + new URL(server.getIssuer()), path.toUri().toURL(), // key file path - AUDIENCE + audience ); admin = spy(PulsarAdmin.builder().serviceHttpUrl(brokerUrl.toString()) @@ -171,6 +128,10 @@ protected final void clientSetup() throws Exception { @Override protected void cleanup() throws Exception { super.internalCleanup(); + } + + @AfterClass(alwaysRun = true) + protected void cleanupAfterClass() { server.stop(); } @@ -292,52 +253,4 @@ public void testOAuth2TokenRefreshedWithoutReconnect() throws Exception { consumer.acknowledgeCumulative(msg); consumer.close(); } - - class OAuth2Transformer extends ResponseTransformer { - - private final PrivateKey privateKey; - private final long tokenTTL; - - OAuth2Transformer(KeyPair key, long tokenTTLMillis) { - this.privateKey = key.getPrivate(); - this.tokenTTL = tokenTTLMillis; - } - - @Override - public Response transform(Request request, Response response, FileSource files, Parameters parameters) { - return Response.Builder.like(response).but().body(""" - { - "access_token": "%s", - "expires_in": %d, - "token_type":"Bearer" - } - """.formatted(generateToken(), - TimeUnit.MILLISECONDS.toSeconds(tokenTTL))).build(); - } - - @Override - public String getName() { - return "o-auth-token-transformer"; - } - - @Override - public boolean applyGlobally() { - return false; - } - - private String generateToken() { - long now = System.currentTimeMillis(); - DefaultJwtBuilder defaultJwtBuilder = new DefaultJwtBuilder(); - defaultJwtBuilder.setHeaderParam("typ", "JWT"); - defaultJwtBuilder.setHeaderParam("alg", "RS256"); - defaultJwtBuilder.setIssuer(server.baseUrl()); - defaultJwtBuilder.setSubject(ADMIN_ROLE); - defaultJwtBuilder.setAudience(AUDIENCE); - defaultJwtBuilder.setIssuedAt(new Date(now)); - defaultJwtBuilder.setNotBefore(new Date(now)); - defaultJwtBuilder.setExpiration(new Date(now + tokenTTL)); - defaultJwtBuilder.signWith(privateKey); - return defaultJwtBuilder.compact(); - } - } } diff --git a/pulsar-broker/src/test/resources/authentication/token/credentials_file.json b/pulsar-broker/src/test/resources/authentication/token/credentials_file.json index db1eccd8eb678..d12e786b7cdb5 100644 --- a/pulsar-broker/src/test/resources/authentication/token/credentials_file.json +++ b/pulsar-broker/src/test/resources/authentication/token/credentials_file.json @@ -1,4 +1,4 @@ { - "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", - "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb" + "client_id":"my-admin-role", + "client_secret":"super-secret-client-secret" } diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 555228d618cb9..1810feb5d346f 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -185,6 +185,12 @@ testcontainers test + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java index d5b70dfa03756..ca35d81d80e4a 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTlsTestWithAuth.java @@ -24,6 +24,7 @@ import java.io.FileWriter; import java.util.Optional; +import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; @@ -42,17 +43,21 @@ public class ProxyTlsTestWithAuth extends MockedPulsarServiceBaseTest { private ProxyService proxyService; private ProxyConfiguration proxyConfig = new ProxyConfiguration(); + private MockOIDCIdentityProvider server; + @Override @BeforeClass protected void setup() throws Exception { internalSetup(); + String clientSecret = "super-secret-client-secret"; + server = new MockOIDCIdentityProvider(clientSecret, "an-audience", 3000); File tempFile = File.createTempFile("oauth2", ".tmp"); tempFile.deleteOnExit(); FileWriter writer = new FileWriter(tempFile); writer.write("{\n" + - " \"client_id\":\"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x\",\n" + - " \"client_secret\":\"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb\"\n" + + " \"client_id\":\"my-user\",\n" + + " \"client_secret\":\"" + clientSecret + "\"\n" + "}"); writer.flush(); writer.close(); @@ -69,8 +74,8 @@ protected void setup() throws Exception { proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); proxyConfig.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"); proxyConfig.setBrokerClientAuthenticationParameters("{\"grant_type\":\"client_credentials\"," + - " \"issuerUrl\":\"https://dev-kt-aa9ne.us.auth0.com\"," + - " \"audience\": \"https://dev-kt-aa9ne.us.auth0.com/api/v2/\"," + + " \"issuerUrl\":\"" + server.getIssuer() + "\"," + + " \"audience\": \"an-audience\"," + " \"privateKey\":\"file://" + tempFile.getAbsolutePath() + "\"}"); proxyService = Mockito.spy(new ProxyService(proxyConfig, new AuthenticationService( @@ -85,8 +90,8 @@ protected void setup() throws Exception { @AfterClass(alwaysRun = true) protected void cleanup() throws Exception { internalCleanup(); - proxyService.close(); + server.stop(); } @Test diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 085931a896af1..b1fdf37e61987 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -112,6 +112,13 @@ test + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java index 05c9b069aca87..d19c4b3d104b7 100644 --- a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/Oauth2PerformanceTransactionTest.java @@ -32,6 +32,7 @@ import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.broker.auth.MockOIDCIdentityProvider; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.Consumer; @@ -42,7 +43,6 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.client.api.TokenOauth2AuthenticatedProducerConsumerTest; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; @@ -61,32 +61,25 @@ public class Oauth2PerformanceTransactionTest extends ProducerConsumerBase { private final String testNamespace = "perf"; private final String myNamespace = testTenant + "/" + testNamespace; private final String testTopic = "persistent://" + myNamespace + "/test-"; - private static final Logger log = LoggerFactory.getLogger(TokenOauth2AuthenticatedProducerConsumerTest.class); - - // public key in oauth2 server to verify the client passed in token. get from https://jwt.io/ - private final String TOKEN_TEST_PUBLIC_KEY = "data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/" - + "4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRm" - + "WjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6" - + "NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDH" - + "y5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB"; - - private final String ADMIN_ROLE = "Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x@clients"; + private static final Logger log = LoggerFactory.getLogger(Oauth2PerformanceTransactionTest.class); // Credentials File, which contains "client_id" and "client_secret" private final String CREDENTIALS_FILE = "./src/test/resources/authentication/token/credentials_file.json"; private final String authenticationPlugin = "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2"; + private MockOIDCIdentityProvider server; private String authenticationParameters; @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { + server = new MockOIDCIdentityProvider("a-client-secret", "my-test-audience", 30000); Path path = Paths.get(CREDENTIALS_FILE).toAbsolutePath(); HashMap params = new HashMap<>(); - params.put("issuerUrl", new URL("https://dev-kt-aa9ne.us.auth0.com")); + params.put("issuerUrl", server.getIssuer()); params.put("privateKey", path.toUri().toURL()); - params.put("audience", "https://dev-kt-aa9ne.us.auth0.com/api/v2/"); + params.put("audience", "my-test-audience"); ObjectMapper jsonMapper = ObjectMapperFactory.create(); authenticationParameters = jsonMapper.writeValueAsString(params); @@ -96,7 +89,7 @@ protected void setup() throws Exception { conf.setAuthenticationRefreshCheckSeconds(5); Set superUserRoles = new HashSet<>(); - superUserRoles.add(ADMIN_ROLE); + superUserRoles.add("superuser"); conf.setSuperUserRoles(superUserRoles); Set providers = new HashSet<>(); @@ -107,7 +100,7 @@ protected void setup() throws Exception { // Set provider domain name Properties properties = new Properties(); - properties.setProperty("tokenPublicKey", TOKEN_TEST_PUBLIC_KEY); + properties.setProperty("tokenPublicKey", server.getBase64EncodedPublicKey()); conf.setProperties(properties); @@ -127,6 +120,7 @@ protected void setup() throws Exception { @Override protected void cleanup() throws Exception { super.internalCleanup(); + server.stop(); } // setup both admin and pulsar client diff --git a/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json b/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json index 698ad9d93e3da..92ab1c3b7cd87 100644 --- a/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json +++ b/pulsar-testclient/src/test/resources/authentication/token/credentials_file.json @@ -1,5 +1,5 @@ { "type": "client_credentials", - "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", - "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb" + "client_id":"superuser", + "client_secret":"a-client-secret" } From ac5527defebb9e6c9fe2802632d6eab573740c73 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 5 Jun 2023 14:48:46 +0300 Subject: [PATCH 134/494] [fix][ci] Fix OWASP dependency check suppressions (#20486) (cherry picked from commit 3b862ae614fae795e3312c0b298cc3c2b33a698f) # Conflicts: # src/owasp-dependency-check-suppressions.xml --- pom.xml | 2 +- src/owasp-dependency-check-suppressions.xml | 47 ++++----------------- 2 files changed, 10 insertions(+), 39 deletions(-) diff --git a/pom.xml b/pom.xml index 47fa6705f1574..58ae756c73516 100644 --- a/pom.xml +++ b/pom.xml @@ -296,7 +296,7 @@ flexible messaging model and an intuitive client API. 0.1.4 1.3 0.4 - 8.1.2 + 8.2.1 0.9.44 1.6.1 6.4.0 diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index dd95cbc1025ef..311204ac37085 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -199,49 +199,20 @@ CVE-2021-42550 - - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15106 - - - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15112 + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:etcd:etcd - - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15113 + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:redhat:etcd - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15106 - - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15112 - - - - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15113 + Ignore grpc CVEs in jetcd + .*jetcd-grpc.* + cpe:/a:grpc:grpc From 398a781a454d5a2f19e3c49c6b917cc9e3cc8fb3 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 6 Jun 2023 09:06:10 +0300 Subject: [PATCH 135/494] Bump version to 3.0.1-SNAPSHOT --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 2 +- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 2 +- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 132 files changed, 132 insertions(+), 132 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index bc7d77ad4f0b4..cf31bd6c64a01 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index cef09439b1e0f..a2eb2d1437082 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 732ade6a10f3e..25b8a32a4b755 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index a93131a7ce43b..5ccb3122dd163 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 9514a0fc4ee78..cbc4dfee05e7a 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,7 +31,7 @@ org.apache.pulsar buildtools - 3.0.0 + 3.0.1-SNAPSHOT jar Pulsar Build Tools diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 72b941f06ab99..7f8f7b33c2e38 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 035cd23e62579..d0c37bd25210b 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 67846dae80084..d9b72f01eaf3f 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index a4e20e62a2fc6..1171f8acaeb8f 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 4ca155f395ae0..a8855daefcecd 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/docker/pom.xml b/docker/pom.xml index 110a3876acaf8..16f905e5ebe67 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index e78eebecc7abf..509084bfada2a 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 5e849f2887e53..3b8a9933cea2b 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index e673ec0404f0a..441f3b11dc5bf 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index d3c5b9c0f4082..6934d1a8432af 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 58ae756c73516..028d6a7dc89da 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index ac9e0a59abc9f..d0035a8b03c96 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-broker-auth-athenz diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index bb50786202301..997739da63cec 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-broker-auth-oidc diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 16d518eae76ce..fdaf75decee2c 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-broker-auth-sasl diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 7ce9e22a81d3a..e66b4dc50f19a 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-broker-common diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 4316ace051a07..b4655d00bac95 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 11fea04a6d364..7614b24050910 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 3ad0cbb4ee287..022a29a91979b 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index ce3055d31e2de..6269a91f60ed9 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-client-1x-base - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 6c09f0f063801..6b0e20b8fb480 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index cdf74cb8eb871..14ce4dea2287b 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index 597db6553ed39..9ca5a32db001c 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 5bdc97c362480..90a334310ea79 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-api/pom.xml b/pulsar-client-api/pom.xml index 4f72e08e6d25e..6e7d2e7a4cd02 100644 --- a/pulsar-client-api/pom.xml +++ b/pulsar-client-api/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index b18157ba85bac..95033a4559968 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-auth-sasl/pom.xml b/pulsar-client-auth-sasl/pom.xml index dbcabe72b143e..6493261f29235 100644 --- a/pulsar-client-auth-sasl/pom.xml +++ b/pulsar-client-auth-sasl/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 60091f3b1ca63..b9a8b0e0f70ad 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 7a5e177cc15f4..456b0a86355c8 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-tools-api/pom.xml b/pulsar-client-tools-api/pom.xml index 48e8572bd6a5d..21a238704ba5e 100644 --- a/pulsar-client-tools-api/pom.xml +++ b/pulsar-client-tools-api/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index 1e76bce988808..cf8ca278b3ed2 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -22,7 +22,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. 4.0.0 diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index a820292007409..570441a4e3221 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index ee74265b2c753..0b6317c8fd86d 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index e43d8415dad98..ed6caef5208f0 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 8e7c8d9ef9a5b..8d64ee4ae3c08 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 3c38bca2c555c..83a3702992e85 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 34a811d04e130..448244b2bbe96 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index fe7a199bd7714..7a536a8303739 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions-instance diff --git a/pulsar-functions/java-examples-builtin/pom.xml b/pulsar-functions/java-examples-builtin/pom.xml index a1d095813f7bd..39c0b25406703 100644 --- a/pulsar-functions/java-examples-builtin/pom.xml +++ b/pulsar-functions/java-examples-builtin/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions-api-examples-builtin diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index f3b0d18ae9269..89a478cbd9eea 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions-api-examples diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 3eab1a772b6d9..e816113ba6c82 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index b9bd58d11d587..1b1904eeac8b0 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 76f602622ef03..f035fd60a22ca 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 36034de2f1d42..f29898514550d 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions-proto diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index a95c5dbfba82f..a6a222c43f0a5 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 689cdac84add2..972872986a2c1 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions-runtime diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index f2786b8c85bd3..9a80767184822 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions-secrets diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index f9d4327c070ea..bae42da7fa80b 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT pulsar-functions-utils diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 803ddfbcb05cc..24bf996d14fe5 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-functions - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index 96f0822f796fb..98227e07c1c27 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-aerospike diff --git a/pulsar-io/alluxio/pom.xml b/pulsar-io/alluxio/pom.xml index d299682867e4f..03aebd3093eb0 100644 --- a/pulsar-io/alluxio/pom.xml +++ b/pulsar-io/alluxio/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index c7a3985bec99e..9f1d34a9ef28a 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-aws diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index cdd61b0ed5d4c..76a36298f2444 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-batch-data-generator diff --git a/pulsar-io/batch-discovery-triggerers/pom.xml b/pulsar-io/batch-discovery-triggerers/pom.xml index a43c836cc6324..7d606e5e7468f 100644 --- a/pulsar-io/batch-discovery-triggerers/pom.xml +++ b/pulsar-io/batch-discovery-triggerers/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-batch-discovery-triggerers diff --git a/pulsar-io/canal/pom.xml b/pulsar-io/canal/pom.xml index afcb6bd31406a..60fa4a4d64a7f 100644 --- a/pulsar-io/canal/pom.xml +++ b/pulsar-io/canal/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 82c22a041ae98..0acdb805aa89e 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-cassandra diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index d006c118c6b25..c889ab209724a 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-common diff --git a/pulsar-io/core/pom.xml b/pulsar-io/core/pom.xml index c6bfd3867253e..f3350829062b2 100644 --- a/pulsar-io/core/pom.xml +++ b/pulsar-io/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-core diff --git a/pulsar-io/data-generator/pom.xml b/pulsar-io/data-generator/pom.xml index 47d7233555fc5..8aaaf0005e95e 100644 --- a/pulsar-io/data-generator/pom.xml +++ b/pulsar-io/data-generator/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-data-generator diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index f90a590d5aead..b4d6ac86bb9e4 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-debezium-core diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 34e2482928deb..855b9b6488e9e 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-debezium-mongodb diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index dceb01a7d9652..f3c485d8fdf8f 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-debezium-mssql diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index ab1ed50f04c93..c98bb5819fb70 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-debezium-mysql diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index f36dfcc49c7d7..2154cfd20d172 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-debezium-oracle diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 9d6992ff043e5..990eaa0278041 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-debezium diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index d2823fca2c845..58f89adcafb91 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io-debezium - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-debezium-postgres diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 559f2d01337db..ea0a91ed99834 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-docs diff --git a/pulsar-io/dynamodb/pom.xml b/pulsar-io/dynamodb/pom.xml index b8a79f21d0b6d..294988ecd9463 100644 --- a/pulsar-io/dynamodb/pom.xml +++ b/pulsar-io/dynamodb/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-dynamodb diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 8d87a9ea68555..5708228f60caa 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-elastic-search Pulsar IO :: ElasticSearch diff --git a/pulsar-io/file/pom.xml b/pulsar-io/file/pom.xml index 46f2fd19969fb..759a340641b06 100644 --- a/pulsar-io/file/pom.xml +++ b/pulsar-io/file/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-file diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 1d5afabc98650..1ca280f6effc2 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-flume diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 78a19f3e43ed1..b98cffa3ce0f9 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-hbase Pulsar IO :: Hbase diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 04ee350408dcd..0121ea7cf0ead 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-hdfs2 Pulsar IO :: Hdfs2 diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index dafcc53b5106d..b8765c193c9b5 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-hdfs3 Pulsar IO :: Hdfs3 diff --git a/pulsar-io/http/pom.xml b/pulsar-io/http/pom.xml index e0d07c3ce53d7..f8b375dbe06b1 100644 --- a/pulsar-io/http/pom.xml +++ b/pulsar-io/http/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-http diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 914cc2d106b00..dd635f3f46d02 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-influxdb diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 56df558bfc4af..85bb55ce95d64 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 4f4fa23d0cc75..c921f402e5a85 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 7e9c7b8461b5a..bf159d93eadc2 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/openmldb/pom.xml b/pulsar-io/jdbc/openmldb/pom.xml index f1f27176e09b1..6d214a06b69ee 100644 --- a/pulsar-io/jdbc/openmldb/pom.xml +++ b/pulsar-io/jdbc/openmldb/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 3590c1fdb1ebf..6dc5fa33a27c8 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -33,7 +33,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-jdbc diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index a15b0ee2824d2..ca27dc246771a 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 29543c4d1c7ed..70abfc124d501 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -24,7 +24,7 @@ pulsar-io-jdbc org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index c870da9d4236a..2162fbfb914db 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-kafka-connect-adaptor-nar diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index b6fc6b428d709..7a5f9070b03d6 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-kafka-connect-adaptor diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 548b247f37718..2d8fc93ed12ff 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-kafka diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index c28fafa2353cc..f8971c741ed5e 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-kinesis diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 51d78863023a4..e9d1d87e3f157 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-mongo diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 05b38952a29e7..22b07af9c68ac 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-netty diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index a574999a255e4..5a5ba73533503 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-nsq diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 0eaa6ad913634..e354ff8e40363 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 72e5d5f9e59c9..a73fa299bca44 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-rabbitmq diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 69fb283b28d97..a1d8051124ccf 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-redis diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 02698a27937b1..7bfea469be331 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -25,7 +25,7 @@ pulsar-io org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index d2702ce1c8c6e..6e789bfdbc2d7 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar-io - 3.0.0 + 3.0.1-SNAPSHOT pulsar-io-twitter diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 8559b38d02004..2d4d1a21a2380 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index 1022fa3fbb5e0..11786444b4839 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 65f70c0c1d038..3f6be6e7bea10 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/filesystem-storage/pom.xml b/pulsar-package-management/filesystem-storage/pom.xml index e2f824d440b00..070c31e1c5680 100644 --- a/pulsar-package-management/filesystem-storage/pom.xml +++ b/pulsar-package-management/filesystem-storage/pom.xml @@ -25,7 +25,7 @@ pulsar-package-management org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index 926117693c992..924d1f868bc66 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. 4.0.0 diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 1810feb5d346f..a053ad1b0329b 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-proxy diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 77fbedeee889a..80b42050fe8fd 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-sql diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index c3727f14eed16..85f19e7f7ca9e 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 3.0.0 + 3.0.1-SNAPSHOT pulsar-presto-distribution diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index 510d9fcd379a5..ab1dbdfd05770 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 3.0.0 + 3.0.1-SNAPSHOT pulsar-presto-connector diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index f34288d1fcb78..c3ea832d8cc54 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar-sql - 3.0.0 + 3.0.1-SNAPSHOT pulsar-presto-connector-original diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index b1fdf37e61987..eb94c72ad3b86 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index 165616ad70f2d..4dee4ace99dfc 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 3.0.0 + 3.0.1-SNAPSHOT pulsar-transaction-common diff --git a/pulsar-transaction/coordinator/pom.xml b/pulsar-transaction/coordinator/pom.xml index 707698d3754ef..639084d9354bc 100644 --- a/pulsar-transaction/coordinator/pom.xml +++ b/pulsar-transaction/coordinator/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar pulsar-transaction-parent - 3.0.0 + 3.0.1-SNAPSHOT pulsar-transaction-coordinator diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index b2808e6e8fb16..6154c192eb53f 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT pulsar-transaction-parent diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index d3425d4f8e690..74891ce908bfc 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index b5c40b06d3405..2df34d7d06ee7 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 3e2d47c59e1a0..13b1a9cda5883 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -25,7 +25,7 @@ pulsar org.apache.pulsar - 3.0.0 + 3.0.1-SNAPSHOT testmocks diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index be09b3244c985..65bf0e8acac83 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0 + 3.0.1-SNAPSHOT bc_2_0_0 diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index d3fa3bbb779d1..4d1766046c415 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0 + 3.0.1-SNAPSHOT bc_2_0_1 diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index b970a5b2320c9..68c7df92b7e2e 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 526620e8995f8..7a764ffcbdbf6 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 java-test-functions diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 8deaed18f9e95..9e88ceac29c27 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 java-test-image diff --git a/tests/docker-images/java-test-plugins/pom.xml b/tests/docker-images/java-test-plugins/pom.xml index 045cec02207c9..6f26b34b021ab 100644 --- a/tests/docker-images/java-test-plugins/pom.xml +++ b/tests/docker-images/java-test-plugins/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 java-test-plugins diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 1b37bc2190c9b..3d6022e41f747 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar.tests docker-images - 3.0.0 + 3.0.1-SNAPSHOT 4.0.0 latest-version-image diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index 50ed707cd0b07..51641494b1a72 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0 + 3.0.1-SNAPSHOT docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 5b6f825f7f975..7ed2bfebfaf4c 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0 + 3.0.1-SNAPSHOT integration diff --git a/tests/pom.xml b/tests/pom.xml index b5ef1a68ad5f7..24de41a5be0ca 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT org.apache.pulsar.tests tests-parent diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 2012cca57dce2..165662aacc351 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0 + 3.0.1-SNAPSHOT pulsar-client-admin-shade-test diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index d2b8df056be58..1bb7a56101733 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0 + 3.0.1-SNAPSHOT pulsar-client-all-shade-test diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 44e307bc1513c..4598ee4243a51 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -27,7 +27,7 @@ org.apache.pulsar.tests tests-parent - 3.0.0 + 3.0.1-SNAPSHOT pulsar-client-shade-test diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 9a19f4eeaaecd..d3e8537e65500 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index d95444e0e8aa1..1edf3db167ef1 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar tiered-storage-parent - 3.0.0 + 3.0.1-SNAPSHOT .. diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index b6359869e31c3..94b47fbc4f794 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.0 + 3.0.1-SNAPSHOT .. From fa63b688e6f4be2de2027b5f4572d3fd728eafeb Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Tue, 6 Jun 2023 19:45:58 +0800 Subject: [PATCH 136/494] [fix][build] Fix the pulsar-all image may use the wrong upstream image (#20435) Signed-off-by: Zike Yang Co-authored-by: Lari Hotari (cherry picked from commit d7f355881b2b1eebf2be6ea262c202660d684fb7) --- docker/build.sh | 2 +- docker/get-version.sh | 2 +- docker/pom.xml | 14 ++++++++++++++ docker/publish.sh | 24 ++++++++++++++++-------- docker/pulsar-all/Dockerfile | 3 ++- docker/pulsar-all/pom.xml | 14 +++++++++++++- docker/pulsar/pom.xml | 17 ++++++++++++++--- pom.xml | 27 ++++++++++++++++++++++++++- pulsar-common/pom.xml | 3 ++- 9 files changed, 89 insertions(+), 17 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index d8ab4bea882c4..88be44f23e73f 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -18,7 +18,7 @@ # under the License. # -ROOT_DIR=$(git rev-parse --show-toplevel) +ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" cd $ROOT_DIR/docker mvn package -Pdocker,-main diff --git a/docker/get-version.sh b/docker/get-version.sh index 07145e7cf0c18..0b736baf3b270 100755 --- a/docker/get-version.sh +++ b/docker/get-version.sh @@ -18,7 +18,7 @@ # under the License. # -ROOT_DIR=$(git rev-parse --show-toplevel) +ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" pushd $ROOT_DIR > /dev/null diff --git a/docker/pom.xml b/docker/pom.xml index 16f905e5ebe67..6e6b27c7d7533 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -60,6 +60,20 @@ pulsar pulsar-all + + + + pl.project13.maven + git-commit-id-plugin + + false + true + true + false + + + + diff --git a/docker/publish.sh b/docker/publish.sh index 45b338d85f8ef..651fefc1498e9 100755 --- a/docker/publish.sh +++ b/docker/publish.sh @@ -18,7 +18,7 @@ # under the License. # -ROOT_DIR=$(git rev-parse --show-toplevel) +ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" cd $ROOT_DIR/docker # We should only publish images that are made from official and approved releases @@ -49,6 +49,9 @@ fi MVN_VERSION=`./get-version.sh` echo "Pulsar version: ${MVN_VERSION}" +GIT_COMMIT_ID_ABBREV=$(git rev-parse --short=7 HEAD 2>/dev/null || echo no-git) +GIT_BRANCH=$(git branch --show-current 2>/dev/null || echo no-git) +IMAGE_TAG="${MVN_VERSION}-${GIT_COMMIT_ID_ABBREV}" if [[ -z ${DOCKER_REGISTRY} ]]; then docker_registry_org=${DOCKER_ORG} @@ -62,16 +65,21 @@ set -x # Fail if any of the subsequent commands fail set -e -docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:latest -docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:latest +if [[ "$GIT_BRANCH" == "master" ]]; then + docker tag apachepulsar/pulsar:${IMAGE_TAG} ${docker_registry_org}/pulsar:latest + docker tag apachepulsar/pulsar-all:${IMAGE_TAG} ${docker_registry_org}/pulsar-all:latest +fi -docker tag apachepulsar/pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION -docker tag apachepulsar/pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION +docker tag apachepulsar/pulsar:${IMAGE_TAG} ${docker_registry_org}/pulsar:$MVN_VERSION +docker tag apachepulsar/pulsar-all:${IMAGE_TAG} ${docker_registry_org}/pulsar-all:$MVN_VERSION # Push all images and tags -docker push ${docker_registry_org}/pulsar:latest -docker push ${docker_registry_org}/pulsar-all:latest +if [[ "$GIT_BRANCH" == "master" ]]; then + docker push ${docker_registry_org}/pulsar:latest + docker push ${docker_registry_org}/pulsar-all:latest +fi + docker push ${docker_registry_org}/pulsar:$MVN_VERSION docker push ${docker_registry_org}/pulsar-all:$MVN_VERSION -echo "Finished pushing images to ${docker_registry_org}" +echo "Finished pushing images to ${docker_registry_org}" \ No newline at end of file diff --git a/docker/pulsar-all/Dockerfile b/docker/pulsar-all/Dockerfile index 42431fc94a067..81ad74b65000f 100644 --- a/docker/pulsar-all/Dockerfile +++ b/docker/pulsar-all/Dockerfile @@ -17,6 +17,7 @@ # under the License. # +ARG PULSAR_IMAGE FROM busybox as pulsar-all ARG PULSAR_IO_DIR @@ -26,6 +27,6 @@ ADD ${PULSAR_IO_DIR} /connectors ADD ${PULSAR_OFFLOADER_TARBALL} / RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders -FROM apachepulsar/pulsar:latest +FROM $PULSAR_IMAGE COPY --from=pulsar-all /connectors /pulsar/connectors COPY --from=pulsar-all /offloaders /pulsar/offloaders diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 509084bfada2a..d4b9e35aff315 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -66,6 +66,17 @@ + + git-commit-id-no-git + + + ${basedir}/../../.git/index + + + + no-git + + docker @@ -144,11 +155,12 @@ ${project.basedir} latest - ${project.version} + ${project.version}-${git.commit.id.abbrev} target/apache-pulsar-io-connectors-${project.version}-bin target/pulsar-offloader-distribution-${project.version}-bin.tar.gz + ${docker.organization}/pulsar:${project.version}-${git.commit.id.abbrev} diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 3b8a9933cea2b..7ec474aca9d89 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -48,11 +48,22 @@ - mirror://mirrors.ubuntu.com/mirrors.txt - http://security.ubuntu.com/ubuntu/ + mirror://mirrors.ubuntu.com/mirrors.txt + http://security.ubuntu.com/ubuntu/ + + git-commit-id-no-git + + + ${basedir}/../../.git/index + + + + no-git + + docker @@ -82,7 +93,7 @@ ${project.basedir} latest - ${project.version} + ${project.version}-${git.commit.id.abbrev} diff --git a/pom.xml b/pom.xml index 028d6a7dc89da..cd82fd9ce3958 100644 --- a/pom.xml +++ b/pom.xml @@ -285,7 +285,7 @@ flexible messaging model and an intuitive client API. 1.1.0 1.5.0 3.1.2 - 4.0.2 + 4.9.10 3.5.3 1.7.0 0.8.8 @@ -1570,6 +1570,31 @@ flexible messaging model and an intuitive client API. + + pl.project13.maven + git-commit-id-plugin + ${git-commit-id-plugin.version} + + + git-info + + revision + + + + + true + true + git + false + false + + false + false + + + + com.mycila license-maven-plugin diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 8d64ee4ae3c08..88daa51d82393 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -194,7 +194,7 @@ provided true - + com.google.protobuf protobuf-java @@ -296,6 +296,7 @@ + false true git false From 1e45f836a8d62aad7516f73189975677223d02d1 Mon Sep 17 00:00:00 2001 From: crossoverJie Date: Thu, 1 Jun 2023 11:48:11 +0800 Subject: [PATCH 137/494] [fix][broker] Fix skip message API when hole messages exists (#20326) (cherry picked from commit c35b820bb323c8e52bd9cd8ccd29565c23764117) --- .../mledger/impl/ManagedCursorImpl.java | 1 - .../pulsar/broker/admin/AdminApiTest.java | 49 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 663081c932052..1ce0403a54762 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1779,7 +1779,6 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) { } finally { if (r.lowerEndpoint() instanceof PositionImplRecyclable) { ((PositionImplRecyclable) r.lowerEndpoint()).recycle(); - ((PositionImplRecyclable) r.upperEndpoint()).recycle(); } } }, recyclePositionRangeConverter); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index f0ac63f08ae4a..e2f5db36a84d0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -55,11 +55,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import javax.ws.rs.client.InvocationCallback; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; import lombok.Builder; import lombok.Cleanup; +import lombok.SneakyThrows; import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; @@ -90,6 +92,7 @@ import org.apache.pulsar.client.api.ConsumerCryptoFailureAction; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; @@ -996,6 +999,52 @@ public void persistentTopics(String topicName) throws Exception { assertEquals(admin.topics().getList("prop-xyz/ns1"), new ArrayList<>()); } + @Test(dataProvider = "topicName") + public void testSkipHoleMessages(String topicName) throws Exception { + final String subName = topicName; + assertEquals(admin.topics().getList("prop-xyz/ns1"), new ArrayList<>()); + + final String persistentTopicName = "persistent://prop-xyz/ns1/" + topicName; + // Force to create a topic + publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 0); + assertEquals(admin.topics().getList("prop-xyz/ns1"), + List.of("persistent://prop-xyz/ns1/" + topicName)); + + // create consumer and subscription + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsar.getWebServiceAddress()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); + AtomicInteger total = new AtomicInteger(); + Consumer consumer = client.newConsumer().topic(persistentTopicName) + .messageListener(new MessageListener() { + @SneakyThrows + @Override + public void received(Consumer consumer, Message msg) { + if (total.get() %2 !=0){ + // artificially created 50 hollow messages + consumer.acknowledge(msg); + } + total.incrementAndGet(); + } + }) + .subscriptionName(subName) + .subscriptionType(SubscriptionType.Exclusive).subscribe(); + + assertEquals(admin.topics().getSubscriptions(persistentTopicName), List.of(subName)); + + publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 100); + TimeUnit.SECONDS.sleep(2); + TopicStats topicStats = admin.topics().getStats(persistentTopicName); + long msgBacklog = topicStats.getSubscriptions().get(subName).getMsgBacklog(); + log.info("back={}",msgBacklog); + int skipNumber = 20; + admin.topics().skipMessages(persistentTopicName, subName, skipNumber); + topicStats = admin.topics().getStats(persistentTopicName); + assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), msgBacklog - skipNumber); + } + @Test(dataProvider = "topicNamesForAllTypes") public void partitionedTopics(String topicType, String topicName) throws Exception { final String namespace = "prop-xyz/ns1"; From d518b71db60125b410ee84acfa36cfabeba89024 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 31 May 2023 09:49:18 +0800 Subject: [PATCH 138/494] [fix][cli] Fulfill add-opens to function-localrunner also (#20417) Confirmed with @RobertIndie . Signed-off-by: tison --- bin/bookkeeper | 2 +- bin/function-localrunner | 14 ++++++++------ bin/pulsar | 2 +- bin/pulsar-admin-common.sh | 2 +- bin/pulsar-perf | 2 +- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/bin/bookkeeper b/bin/bookkeeper index fb516a98acdc2..0cc07dd49aba5 100755 --- a/bin/bookkeeper +++ b/bin/bookkeeper @@ -168,7 +168,7 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $BOOKIE_LOG_CONF`" # Allow Netty to use reflection access OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true" -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` +IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' ) # Start --add-opens options # '--add-opens' option is not supported in jdk8 if [[ -z "$IS_JAVA_8" ]]; then diff --git a/bin/function-localrunner b/bin/function-localrunner index 45a37cb306794..2e0aa0f6dffe2 100755 --- a/bin/function-localrunner +++ b/bin/function-localrunner @@ -40,13 +40,15 @@ PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"} PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"} # Garbage collection log. -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` -# java version has space, use [[ -n $PARAM ]] to judge if variable exists -if [[ -n "$IS_JAVA_8" ]]; then - PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xloggc:logs/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"} -else -# After jdk 9, gc log param should config like this. Ignoring version less than jdk 8 +IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' ) +if [[ -z "$IS_JAVA_8" ]]; then + # >= JDK 9 PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xlog:gc:logs/pulsar_gc_%p.log:time,uptime:filecount=10,filesize=20M"} + # '--add-opens' option is not supported in JDK 1.8 + OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED" +else + # == JDK 1.8 + PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xloggc:logs/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"} fi # Extra options to be passed to the jvm diff --git a/bin/pulsar b/bin/pulsar index a033de947d4b3..9d924bb296c59 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -291,7 +291,7 @@ OPTS="$OPTS -Dzookeeper.clientTcpKeepAlive=true" # Allow Netty to use reflection access OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true" -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` +IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' ) # Start --add-opens options # '--add-opens' option is not supported in jdk8 if [[ -z "$IS_JAVA_8" ]]; then diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh index 8223ac5b3bf24..8aa21c00f634d 100755 --- a/bin/pulsar-admin-common.sh +++ b/bin/pulsar-admin-common.sh @@ -91,7 +91,7 @@ PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH" OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`" OPTS="$OPTS -Djava.net.preferIPv4Stack=true" -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` +IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' ) # Start --add-opens options # '--add-opens' option is not supported in jdk8 if [[ -z "$IS_JAVA_8" ]]; then diff --git a/bin/pulsar-perf b/bin/pulsar-perf index 47c02bc3d67d5..bdc1dc1ed8b8c 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -134,7 +134,7 @@ PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH" PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH" OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF` -Djava.net.preferIPv4Stack=true" -IS_JAVA_8=`$JAVA -version 2>&1 |grep version|grep '"1\.8'` +IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' ) # Start --add-opens options # '--add-opens' option is not supported in jdk8 if [[ -z "$IS_JAVA_8" ]]; then From 67b1fe86e016f9fa92d9fbfbd911f0a0d3e5e6b2 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Tue, 6 Jun 2023 21:46:46 -0500 Subject: [PATCH 139/494] [fix][fn] TLS args admin download command use zero arity (#20513) ### Motivation #20482 broke the function download command with this error: ``` Expected a command, got false Usage: pulsar-admin [options] [command] [command options] Options: --admin-url Admin Service URL to which to connect. Default: http://localhost:8080/ ``` The problem is that the TLS args for hostname verification and for insecure TLS are zero arity, and therefore, we should not add the `false` or the `true` arguments. ### Modifications * Correct the changes made to the TLS args passed to the `pulsar-admin` CLI tool ### Documentation - [x] `doc-not-needed` (cherry picked from commit 50b9a93e42e412d9f17b1637287d1a4c7c7ab148) --- .../runtime/kubernetes/KubernetesRuntime.java | 11 ++++++----- .../runtime/kubernetes/KubernetesRuntimeTest.java | 4 +--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 7779422965ba4..36c5a17ee6595 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -885,11 +885,12 @@ && isNotBlank(authConfig.getClientAuthenticationParameters())) { "--auth-params", authConfig.getClientAuthenticationParameters())); } - cmd.addAll(Arrays.asList( - "--tls-allow-insecure", - Boolean.toString(authConfig.isTlsAllowInsecureConnection()), - "--tls-enable-hostname-verification", - Boolean.toString(authConfig.isTlsHostnameVerificationEnable()))); + if (authConfig.isTlsAllowInsecureConnection()) { + cmd.add("--tls-allow-insecure"); + } + if (authConfig.isTlsHostnameVerificationEnable()) { + cmd.add("--tls-enable-hostname-verification"); + } if (isNotBlank(authConfig.getTlsTrustCertsFilePath())) { cmd.addAll(Arrays.asList( "--tls-trust-cert-path", diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 2cbbeb8a00b92..31f82adfe8c26 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -852,7 +852,6 @@ public void testCustomKubernetesDownloadCommandsWithAuth() throws Exception { V1StatefulSet spec = container.createStatefulSet(); String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}" - + " --tls-allow-insecure false --tls-enable-hostname-verification false" + " functions download " + "--tenant " + TEST_TENANT + " --namespace " + TEST_NAMESPACE @@ -879,7 +878,6 @@ public void testCustomKubernetesDownloadCommandsWithAuthWithoutAuthSpec() throws V1StatefulSet spec = container.createStatefulSet(); String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}" - + " --tls-allow-insecure false --tls-enable-hostname-verification false" + " functions download " + "--tenant " + TEST_TENANT + " --namespace " + TEST_NAMESPACE @@ -909,7 +907,7 @@ public void testCustomKubernetesDownloadCommandsWithAuthAndCustomTLSWithoutAuthS V1StatefulSet spec = container.createStatefulSet(); String expectedDownloadCommand = "pulsar-admin --admin-url " + pulsarAdminUrl + " --auth-plugin com.MyAuth --auth-params {\"authParam1\": \"authParamValue1\"}" - + " --tls-allow-insecure false --tls-enable-hostname-verification true" + + " --tls-enable-hostname-verification" + " --tls-trust-cert-path /my/ca.pem" + " functions download " + "--tenant " + TEST_TENANT From ec7b89e974ae3976dae2ed14b1087849c8d2d4b5 Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Wed, 31 May 2023 14:34:32 +0800 Subject: [PATCH 140/494] [fix] [broker] do not filter system topic while shedding. (#18949) ### Motivation Currently, topics/bundles in `pulsar/system` will be filter while doing shedding, which is introduced by mistake by pr https://github.com/apache/pulsar/pull/15252. But we need to unload topics/bundles in `pulsar/system` for load balancing. ### Modifications do not filter topics/bundles in `pulsar/system`. (cherry picked from commit 5c74d207426e26ec4feea279ed2ae1b1c2909cba) --- .../apache/pulsar/broker/loadbalance/LoadData.java | 2 +- .../pulsar/broker/namespace/NamespaceService.java | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java index 87f630f1a09fb..a632a47f05116 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java @@ -64,7 +64,7 @@ public Map getBundleData() { public Map getBundleDataForLoadShedding() { return bundleData.entrySet().stream() - .filter(e -> !NamespaceService.isSystemServiceNamespace( + .filter(e -> !NamespaceService.filterNamespaceForShedding( NamespaceBundle.getBundleNamespace(e.getKey()))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 9d8d9e3890a19..e2d4ef5153769 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1607,6 +1607,17 @@ public static boolean isSystemServiceNamespace(String namespace) { || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches(); } + /** + * used for filtering bundles in special namespace. + * @param namespace + * @return True if namespace is HEARTBEAT_NAMESPACE or SLA_NAMESPACE + */ + public static boolean filterNamespaceForShedding(String namespace) { + return SLA_NAMESPACE_PATTERN.matcher(namespace).matches() + || HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches() + || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches(); + } + public static boolean isHeartbeatNamespace(ServiceUnitId ns) { String namespace = ns.getNamespaceObject().toString(); return HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches() From adc931a422cbd76e17664f3e7529784f3e07a72e Mon Sep 17 00:00:00 2001 From: Rajan Dhabalia Date: Tue, 6 Jun 2023 03:58:40 -0700 Subject: [PATCH 141/494] [fix][cli] Fix logging noise while admin tool exit (#19884) Co-authored-by: tison (cherry picked from commit ad352a90f13173324feac816ee97ffff5140cf8f) --- .../java/org/apache/pulsar/admin/cli/PulsarAdminTool.java | 2 +- .../java/org/apache/pulsar/common/util/ShutdownUtil.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index ca0a8a055cfc9..6844504223989 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -312,7 +312,7 @@ private static void exit(int code) { if (allowSystemExit) { // we are using halt and not System.exit, we do not mind about shutdown hooks // they are only slowing down the tool - ShutdownUtil.triggerImmediateForcefulShutdown(code); + ShutdownUtil.triggerImmediateForcefulShutdown(code, false); } else { System.out.println("Exit code is " + code + " (System.exit not called, as we are in test mode)"); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java index b461cfab0d1bc..ff12484707123 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/ShutdownUtil.java @@ -47,8 +47,11 @@ public class ShutdownUtil { * @see Runtime#halt(int) */ public static void triggerImmediateForcefulShutdown(int status) { + triggerImmediateForcefulShutdown(status, true); + } + public static void triggerImmediateForcefulShutdown(int status, boolean logging) { try { - if (status != 0) { + if (status != 0 && logging) { log.warn("Triggering immediate shutdown of current process with status {}", status, new Exception("Stacktrace for immediate shutdown")); } From 78e033c24ab87f407b14bc743b04f99cc83db076 Mon Sep 17 00:00:00 2001 From: crossoverJie Date: Mon, 5 Jun 2023 16:10:44 +0800 Subject: [PATCH 142/494] [fix][client] Fix where the function getMsgNumInReceiverQueue always returns 0 when using message listener (#20245) (cherry picked from commit 2dec2677743a6c517deb6976718fbe3a9eef0644) --- .../api/SimpleProducerConsumerTest.java | 29 +++++++++++++++++++ .../pulsar/client/impl/ConsumerBase.java | 8 +++++ .../impl/ConsumerStatsRecorderImpl.java | 6 +++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index f3a00531eba56..c45c2b1522f59 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -4109,6 +4109,35 @@ public void testGetStats() throws Exception { consumer.close(); producer.close(); } + @Test(timeOut = 100000) + public void testMessageListenerGetStats() throws Exception { + final String topicName = "persistent://my-property/my-ns/testGetStats" + UUID.randomUUID(); + final String subName = "my-sub"; + final int receiveQueueSize = 100; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) + .messageListener((MessageListener) (consumer1, msg) -> { + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException igr) { + } + }) + .topic(topicName).receiverQueueSize(receiveQueueSize).subscriptionName(subName).subscribe(); + Assert.assertNull(consumer.getStats().getMsgNumInSubReceiverQueue()); + Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), 0); + + for (int i = 0; i < receiveQueueSize; i++) { + producer.sendAsync("msg" + i); + } + //Give some time to consume + Awaitility.await() + .untilAsserted(() -> Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), receiveQueueSize)); + consumer.close(); + producer.close(); + } @Test(timeOut = 100000) public void testGetStatsForPartitionedTopic() throws Exception { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 0db2a8e0ab9f5..e933005f2d6ea 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -88,6 +88,11 @@ public abstract class ConsumerBase extends HandlerState implements Consumer>> pendingReceives; protected final int maxReceiverQueueSize; private volatile int currentReceiverQueueSize; + + protected static final AtomicIntegerFieldUpdater MESSAGE_LISTENER_QUEUE_SIZE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(ConsumerBase.class, "messageListenerQueueSize"); + protected volatile int messageListenerQueueSize = 0; + protected static final AtomicIntegerFieldUpdater CURRENT_RECEIVER_QUEUE_SIZE_UPDATER = AtomicIntegerFieldUpdater.newUpdater(ConsumerBase.class, "currentReceiverQueueSize"); protected final Schema schema; @@ -1105,6 +1110,7 @@ private void triggerListener() { // Trigger the notification on the message listener in a separate thread to avoid blocking the // internal pinned executor thread while the message processing happens final Message finalMsg = msg; + MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.incrementAndGet(this); if (SubscriptionType.Key_Shared == conf.getSubscriptionType()) { executorProvider.getExecutor(peekMessageKey(msg)).execute(() -> callMessageListener(finalMsg)); @@ -1148,6 +1154,8 @@ protected void callMessageListener(Message msg) { } catch (Throwable t) { log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription, msg.getMessageId(), t); + } finally { + MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.decrementAndGet(this); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java index 8630bedc65f31..3a47ddc5d4b08 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerStatsRecorderImpl.java @@ -238,7 +238,11 @@ public void updateCumulativeStats(ConsumerStats stats) { @Override public Integer getMsgNumInReceiverQueue() { if (consumer instanceof ConsumerBase) { - return ((ConsumerBase) consumer).incomingMessages.size(); + ConsumerBase consumerBase = (ConsumerBase) consumer; + if (consumerBase.listener != null){ + return ConsumerBase.MESSAGE_LISTENER_QUEUE_SIZE_UPDATER.get(consumerBase); + } + return consumerBase.incomingMessages.size(); } return null; } From 223dbf41c1f2b506fc2103a1feab3fe571c8ed0f Mon Sep 17 00:00:00 2001 From: YingQun Zhong Date: Wed, 31 May 2023 15:58:51 +0800 Subject: [PATCH 143/494] [fix][offload] fix offload metrics error (#20366) (cherry picked from commit 8e0ebba345f5f02bf2434fec9a75268d92b1b60d) --- .../pulsar/common/naming/TopicName.java | 36 +++++++++++++++++++ .../pulsar/common/naming/TopicNameTest.java | 34 ++++++++++++++++++ .../impl/FileStoreBackedReadHandleImpl.java | 11 +++--- .../FileSystemManagedLedgerOffloader.java | 22 ++++++++---- .../FileSystemManagedLedgerOffloaderTest.java | 24 +++++++------ .../impl/BlobStoreBackedInputStreamImpl.java | 9 +++-- .../impl/BlobStoreBackedReadHandleImpl.java | 4 ++- .../impl/BlobStoreBackedReadHandleImplV2.java | 4 ++- .../impl/BlobStoreManagedLedgerOffloader.java | 12 ++++--- .../BlockAwareSegmentInputStreamImpl.java | 7 ++-- .../BlobStoreManagedLedgerOffloaderTest.java | 9 ++--- 11 files changed, 135 insertions(+), 37 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java index bab9c443ceaeb..69d3aaeeebb4d 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/naming/TopicName.java @@ -339,6 +339,42 @@ public String getPersistenceNamingEncoding() { } } + /** + * get topic full name from managedLedgerName. + * + * @return the topic full name, format -> domain://tenant/namespace/topic + */ + public static String fromPersistenceNamingEncoding(String mlName) { + // The managedLedgerName convention is: tenant/namespace/domain/topic + // We want to transform to topic full name in the order: domain://tenant/namespace/topic + if (mlName == null || mlName.length() == 0) { + return mlName; + } + List parts = Splitter.on("/").splitToList(mlName); + String tenant; + String cluster; + String namespacePortion; + String domain; + String localName; + if (parts.size() == 4) { + tenant = parts.get(0); + cluster = null; + namespacePortion = parts.get(1); + domain = parts.get(2); + localName = parts.get(3); + return String.format("%s://%s/%s/%s", domain, tenant, namespacePortion, localName); + } else if (parts.size() == 5) { + tenant = parts.get(0); + cluster = parts.get(1); + namespacePortion = parts.get(2); + domain = parts.get(3); + localName = parts.get(4); + return String.format("%s://%s/%s/%s/%s", domain, tenant, cluster, namespacePortion, localName); + } else { + throw new IllegalArgumentException("Invalid managedLedger name: " + mlName); + } + } + /** * Get a string suitable for completeTopicName lookup. * diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java index 0e4533e7a08d0..6d3ef7599d30e 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/naming/TopicNameTest.java @@ -236,6 +236,40 @@ public void testDecodeEncode() throws Exception { assertEquals(name.getPersistenceNamingEncoding(), "prop/colo/ns/persistent/" + encodedName); } + @Test + public void testFromPersistenceNamingEncoding() { + // case1: V2 + String mlName1 = "public_tenant/default_namespace/persistent/test_topic"; + String expectedTopicName1 = "persistent://public_tenant/default_namespace/test_topic"; + + TopicName name1 = TopicName.get(expectedTopicName1); + assertEquals(name1.getPersistenceNamingEncoding(), mlName1); + assertEquals(TopicName.fromPersistenceNamingEncoding(mlName1), expectedTopicName1); + + // case2: V1 + String mlName2 = "public_tenant/my_cluster/default_namespace/persistent/test_topic"; + String expectedTopicName2 = "persistent://public_tenant/my_cluster/default_namespace/test_topic"; + + TopicName name2 = TopicName.get(expectedTopicName2); + assertEquals(name2.getPersistenceNamingEncoding(), mlName2); + assertEquals(TopicName.fromPersistenceNamingEncoding(mlName2), expectedTopicName2); + + // case3: null + String mlName3 = ""; + String expectedTopicName3 = ""; + assertEquals(expectedTopicName3, TopicName.fromPersistenceNamingEncoding(mlName3)); + + // case4: Invalid name + try { + String mlName4 = "public_tenant/my_cluster/default_namespace/persistent/test_topic/sub_topic"; + TopicName.fromPersistenceNamingEncoding(mlName4); + fail("Should have raised exception"); + } catch (IllegalArgumentException e) { + // Exception is expected. + } + } + + @SuppressWarnings("deprecation") @Test public void testTopicNameWithoutCluster() throws Exception { diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java index bdeb88e9ac93c..506fbb8de68bf 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java @@ -40,6 +40,7 @@ import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; +import org.apache.pulsar.common.naming.TopicName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +52,7 @@ public class FileStoreBackedReadHandleImpl implements ReadHandle { private final LedgerMetadata ledgerMetadata; private final LedgerOffloaderStats offloaderStats; private final String managedLedgerName; + private final String topicName; private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader reader, long ledgerId, LedgerOffloaderStats offloaderStats, @@ -60,13 +62,14 @@ private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader r this.reader = reader; this.offloaderStats = offloaderStats; this.managedLedgerName = managedLedgerName; + this.topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); try { key.set(FileSystemManagedLedgerOffloader.METADATA_KEY_INDEX); long startReadIndexTime = System.nanoTime(); reader.get(key, value); - offloaderStats.recordReadOffloadIndexLatency(managedLedgerName, + offloaderStats.recordReadOffloadIndexLatency(topicName, System.nanoTime() - startReadIndexTime, TimeUnit.NANOSECONDS); this.ledgerMetadata = parseLedgerMetadata(ledgerId, value.copyBytes()); } catch (IOException e) { @@ -125,7 +128,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr while (entriesToRead > 0) { long startReadTime = System.nanoTime(); reader.next(key, value); - this.offloaderStats.recordReadOffloadDataLatency(managedLedgerName, + this.offloaderStats.recordReadOffloadDataLatency(topicName, System.nanoTime() - startReadTime, TimeUnit.NANOSECONDS); int length = value.getLength(); long entryId = key.get(); @@ -135,7 +138,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr buf.writeBytes(value.copyBytes()); entriesToRead--; nextExpectedId++; - this.offloaderStats.recordReadOffloadBytes(managedLedgerName, length); + this.offloaderStats.recordReadOffloadBytes(topicName, length); } else if (entryId > lastEntry) { log.info("Expected to read {}, but read {}, which is greater than last entry {}", nextExpectedId, entryId, lastEntry); @@ -144,7 +147,7 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr } promise.complete(LedgerEntriesImpl.create(entries)); } catch (Throwable t) { - this.offloaderStats.recordReadOffloadError(managedLedgerName); + this.offloaderStats.recordReadOffloadError(topicName); promise.completeExceptionally(t); entries.forEach(LedgerEntry::close); } diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java index d5e09ba725421..25b63374946c8 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java @@ -45,6 +45,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -197,9 +198,10 @@ public void run() { return; } long ledgerId = readHandle.getId(); - final String topicName = extraMetadata.get(MANAGED_LEDGER_NAME); - String storagePath = getStoragePath(storageBasePath, topicName); + final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME); + String storagePath = getStoragePath(storageBasePath, managedLedgerName); String dataFilePath = getDataFilePath(storagePath, ledgerId, uuid); + final String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); try { @@ -241,7 +243,7 @@ public void run() { promise.complete(null); } catch (Exception e) { log.error("Exception when get CompletableFuture : ManagerLedgerName: {}, " - + "LedgerId: {}, UUID: {} ", topicName, ledgerId, uuid, e); + + "LedgerId: {}, UUID: {} ", managedLedgerName, ledgerId, uuid, e); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -306,22 +308,27 @@ public static FileSystemWriter create(LedgerEntries ledgerEntriesOnce, @Override public void run() { String managedLedgerName = ledgerReader.extraMetadata.get(MANAGED_LEDGER_NAME); + String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); if (ledgerReader.fileSystemWriteException == null) { Iterator iterator = ledgerEntriesOnce.iterator(); while (iterator.hasNext()) { LedgerEntry entry = iterator.next(); long entryId = entry.getEntryId(); key.set(entryId); + byte[] currentEntryBytes; + int currentEntrySize; try { - value.set(entry.getEntryBytes(), 0, entry.getEntryBytes().length); + currentEntryBytes = entry.getEntryBytes(); + currentEntrySize = currentEntryBytes.length; + value.set(currentEntryBytes, 0, currentEntrySize); dataWriter.append(key, value); } catch (IOException e) { ledgerReader.fileSystemWriteException = e; - ledgerReader.offloaderStats.recordWriteToStorageError(managedLedgerName); + ledgerReader.offloaderStats.recordWriteToStorageError(topicName); break; } haveOffloadEntryNumber.incrementAndGet(); - ledgerReader.offloaderStats.recordOffloadBytes(managedLedgerName, entry.getLength()); + ledgerReader.offloaderStats.recordOffloadBytes(topicName, currentEntrySize); } } countDownLatch.countDown(); @@ -367,6 +374,7 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map promise = new CompletableFuture<>(); try { fileSystem.delete(new Path(dataFilePath), true); @@ -376,7 +384,7 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, Map - this.offloaderStats.recordDeleteOffloadOps(ledgerName, t == null)); + this.offloaderStats.recordDeleteOffloadOps(topicName, t == null)); } @Override diff --git a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java index 1aebab571c971..b9de5d1a49e9a 100644 --- a/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java +++ b/tiered-storage/file-system/src/test/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloaderTest.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.pulsar.common.naming.TopicName; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.net.URI; @@ -46,8 +47,9 @@ public class FileSystemManagedLedgerOffloaderTest extends FileStoreTestBase { private final PulsarMockBookKeeper bk; - private String topic = "public/default/testOffload"; - private String storagePath = createStoragePath(topic); + private String managedLedgerName = "public/default/persistent/testOffload"; + private String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); + private String storagePath = createStoragePath(managedLedgerName); private LedgerHandle lh; private ReadHandle toWrite; private final int numberOfEntries = 601; @@ -56,7 +58,7 @@ public class FileSystemManagedLedgerOffloaderTest extends FileStoreTestBase { public FileSystemManagedLedgerOffloaderTest() throws Exception { this.bk = new PulsarMockBookKeeper(scheduler); this.toWrite = buildReadHandle(); - map.put("ManagedLedgerName", topic); + map.put("ManagedLedgerName", managedLedgerName); } private ReadHandle buildReadHandle() throws Exception { @@ -125,10 +127,10 @@ public void testOffloadAndReadMetrics() throws Exception { offloader.offload(toWrite, uuid, map).get(); LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) this.offloaderStats; - assertTrue(offloaderStats.getOffloadError(topic) == 0); - assertTrue(offloaderStats.getOffloadBytes(topic) > 0); - assertTrue(offloaderStats.getReadLedgerLatency(topic).count > 0); - assertTrue(offloaderStats.getWriteStorageError(topic) == 0); + assertTrue(offloaderStats.getOffloadError(topicName) == 0); + assertTrue(offloaderStats.getOffloadBytes(topicName) > 0); + assertTrue(offloaderStats.getReadLedgerLatency(topicName).count > 0); + assertTrue(offloaderStats.getWriteStorageError(topicName) == 0); ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, map).get(); LedgerEntries toTestEntries = toTest.read(0, numberOfEntries - 1); @@ -137,10 +139,10 @@ public void testOffloadAndReadMetrics() throws Exception { LedgerEntry toTestEntry = toTestIter.next(); } - assertTrue(offloaderStats.getReadOffloadError(topic) == 0); - assertTrue(offloaderStats.getReadOffloadBytes(topic) > 0); - assertTrue(offloaderStats.getReadOffloadDataLatency(topic).count > 0); - assertTrue(offloaderStats.getReadOffloadIndexLatency(topic).count > 0); + assertTrue(offloaderStats.getReadOffloadError(topicName) == 0); + assertTrue(offloaderStats.getReadOffloadBytes(topicName) > 0); + assertTrue(offloaderStats.getReadOffloadDataLatency(topicName).count > 0); + assertTrue(offloaderStats.getReadOffloadIndexLatency(topicName).count > 0); } @Test diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java index aa27df46c5e65..0dea46726f50a 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedInputStreamImpl.java @@ -26,6 +26,7 @@ import org.apache.bookkeeper.mledger.offload.jcloud.BackedInputStream; import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.naming.TopicName; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.options.GetOptions; @@ -44,6 +45,7 @@ public class BlobStoreBackedInputStreamImpl extends BackedInputStream { private final int bufferSize; private LedgerOffloaderStats offloaderStats; private String managedLedgerName; + private String topicName; private long cursor; private long bufferOffsetStart; @@ -71,6 +73,7 @@ public BlobStoreBackedInputStreamImpl(BlobStore blobStore, String bucket, String this(blobStore, bucket, key, versionCheck, objectLen, bufferSize); this.offloaderStats = offloaderStats; this.managedLedgerName = managedLedgerName; + this.topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); } /** @@ -110,13 +113,13 @@ private boolean refillBufferIfNeeded() throws IOException { // because JClouds streams the content // and actually the HTTP call finishes when the stream is fully read if (this.offloaderStats != null) { - this.offloaderStats.recordReadOffloadDataLatency(managedLedgerName, + this.offloaderStats.recordReadOffloadDataLatency(topicName, System.nanoTime() - startReadTime, TimeUnit.NANOSECONDS); - this.offloaderStats.recordReadOffloadBytes(managedLedgerName, endRange - startRange + 1); + this.offloaderStats.recordReadOffloadBytes(topicName, endRange - startRange + 1); } } catch (Throwable e) { if (null != this.offloaderStats) { - this.offloaderStats.recordReadOffloadError(this.managedLedgerName); + this.offloaderStats.recordReadOffloadError(this.topicName); } throw new IOException("Error reading from BlobStore", e); } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java index cdabe5ece0ba2..5a571bb208e34 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java @@ -45,6 +45,7 @@ import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockBuilder; import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.naming.TopicName; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.slf4j.Logger; @@ -261,6 +262,7 @@ public static ReadHandle open(ScheduledExecutorService executor, int retryCount = 3; OffloadIndexBlock index = null; IOException lastException = null; + String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); // The following retry is used to avoid to some network issue cause read index file failure. // If it can not recovery in the retry, we will throw the exception and the dispatcher will schedule to // next read. @@ -269,7 +271,7 @@ public static ReadHandle open(ScheduledExecutorService executor, while (retryCount-- > 0) { long readIndexStartTime = System.nanoTime(); Blob blob = blobStore.getBlob(bucket, indexKey); - offloaderStats.recordReadOffloadIndexLatency(managedLedgerName, + offloaderStats.recordReadOffloadIndexLatency(topicName, System.nanoTime() - readIndexStartTime, TimeUnit.NANOSECONDS); versionCheck.check(indexKey, blob); OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create(); diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java index 2e3d0b08970ca..e40a0a3834c85 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java @@ -46,6 +46,7 @@ import org.apache.bookkeeper.mledger.offload.jcloud.OffloadIndexBlockV2Builder; import org.apache.bookkeeper.mledger.offload.jcloud.impl.DataBlockUtils.VersionCheck; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.naming.TopicName; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; import org.slf4j.Logger; @@ -297,13 +298,14 @@ public static ReadHandle open(ScheduledExecutorService executor, throws IOException { List inputStreams = new LinkedList<>(); List indice = new LinkedList<>(); + String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); for (int i = 0; i < indexKeys.size(); i++) { String indexKey = indexKeys.get(i); String key = keys.get(i); log.debug("open bucket: {} index key: {}", bucket, indexKey); long startTime = System.nanoTime(); Blob blob = blobStore.getBlob(bucket, indexKey); - offloaderStats.recordReadOffloadIndexLatency(managedLedgerName, + offloaderStats.recordReadOffloadIndexLatency(topicName, System.nanoTime() - startTime, TimeUnit.NANOSECONDS); log.debug("indexKey blob: {} {}", indexKey, blob); versionCheck.check(indexKey, blob); diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java index 6d69b5edbc3fb..7a15c414aa8c4 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java @@ -63,6 +63,7 @@ import org.apache.bookkeeper.mledger.offload.jcloud.provider.BlobStoreLocation; import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration; import org.apache.bookkeeper.mledger.proto.MLDataFormats; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.domain.Blob; @@ -176,7 +177,8 @@ public Map getOffloadDriverMetadata() { public CompletableFuture offload(ReadHandle readHandle, UUID uuid, Map extraMetadata) { - final String topicName = extraMetadata.get(MANAGED_LEDGER_NAME); + final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME); + final String topicName = TopicName.fromPersistenceNamingEncoding(managedLedgerName); final BlobStore writeBlobStore = blobStores.get(config.getBlobStoreLocation()); log.info("offload {} uuid {} extraMetadata {} to {} {}", readHandle.getId(), uuid, extraMetadata, config.getBlobStoreLocation(), writeBlobStore); @@ -226,7 +228,7 @@ public CompletableFuture offload(ReadHandle readHandle, .calculateBlockSize(config.getMaxBlockSizeInBytes(), readHandle, startEntry, entryBytesWritten); try (BlockAwareSegmentInputStream blockStream = new BlockAwareSegmentInputStreamImpl( - readHandle, startEntry, blockSize, this.offloaderStats, topicName)) { + readHandle, startEntry, blockSize, this.offloaderStats, managedLedgerName)) { Payload partPayload = Payloads.newInputStreamPayload(blockStream); partPayload.getContentMetadata().setContentLength((long) blockSize); @@ -611,7 +613,8 @@ public CompletableFuture deleteOffloaded(long ledgerId, UUID uid, return promise.whenComplete((__, t) -> { if (null != this.ml) { - this.offloaderStats.recordDeleteOffloadOps(this.ml.getName(), t == null); + this.offloaderStats.recordDeleteOffloadOps( + TopicName.fromPersistenceNamingEncoding(this.ml.getName()), t == null); } }); } @@ -636,7 +639,8 @@ public CompletableFuture deleteOffloaded(UUID uid, Map off }); return promise.whenComplete((__, t) -> - this.offloaderStats.recordDeleteOffloadOps(this.ml.getName(), t == null)); + this.offloaderStats.recordDeleteOffloadOps( + TopicName.fromPersistenceNamingEncoding(this.ml.getName()), t == null)); } @Override diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java index d07fbdb92477b..06d7f2129ba31 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlockAwareSegmentInputStreamImpl.java @@ -36,6 +36,7 @@ import org.apache.bookkeeper.mledger.LedgerOffloaderStats; import org.apache.bookkeeper.mledger.offload.jcloud.BlockAwareSegmentInputStream; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; +import org.apache.pulsar.common.naming.TopicName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +74,7 @@ public class BlockAwareSegmentInputStreamImpl extends BlockAwareSegmentInputStre // Keep a list of all entries ByteBuf, each ByteBuf contains 2 buf: entry header and entry content. private List entriesByteBuf = null; private LedgerOffloaderStats offloaderStats; + private String managedLedgerName; private String topicName; private int currentOffset = 0; private final AtomicBoolean close = new AtomicBoolean(false); @@ -91,7 +93,8 @@ public BlockAwareSegmentInputStreamImpl(ReadHandle ledger, long startEntryId, in LedgerOffloaderStats offloaderStats, String ledgerName) { this(ledger, startEntryId, blockSize); this.offloaderStats = offloaderStats; - this.topicName = ledgerName; + this.managedLedgerName = ledgerName; + this.topicName = TopicName.fromPersistenceNamingEncoding(ledgerName); } private ByteBuf readEntries(int len) throws IOException { @@ -183,7 +186,7 @@ private List readNextEntriesFromLedger(long start, long maxNumberEntrie log.debug("read ledger entries. start: {}, end: {} cost {}", start, end, TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - startTime)); } - if (offloaderStats != null && topicName != null) { + if (offloaderStats != null && managedLedgerName != null) { offloaderStats.recordReadLedgerLatency(topicName, System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } diff --git a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java index ef0bea29e35c3..ac87a8e424038 100644 --- a/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java +++ b/tiered-storage/jcloud/src/test/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloaderTest.java @@ -49,6 +49,7 @@ import org.apache.bookkeeper.mledger.OffloadedLedgerMetadata; import org.apache.bookkeeper.mledger.offload.jcloud.provider.JCloudBlobStoreProvider; import org.apache.bookkeeper.mledger.offload.jcloud.provider.TieredStorageConfiguration; +import org.apache.pulsar.common.naming.TopicName; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.options.CopyOptions; import org.mockito.Mockito; @@ -172,10 +173,10 @@ public void testOffloadAndReadMetrics() throws Exception { LedgerOffloader offloader = getOffloader(); UUID uuid = UUID.randomUUID(); - - String topic = "test"; + String managedLegerName = "public/default/persistent/testOffload"; + String topic = TopicName.fromPersistenceNamingEncoding(managedLegerName); Map extraMap = new HashMap<>(); - extraMap.put("ManagedLedgerName", topic); + extraMap.put("ManagedLedgerName", managedLegerName); offloader.offload(toWrite, uuid, extraMap).get(); LedgerOffloaderStatsImpl offloaderStats = (LedgerOffloaderStatsImpl) this.offloaderStats; @@ -187,7 +188,7 @@ public void testOffloadAndReadMetrics() throws Exception { Map map = new HashMap<>(); map.putAll(offloader.getOffloadDriverMetadata()); - map.put("ManagedLedgerName", topic); + map.put("ManagedLedgerName", managedLegerName); ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, map).get(); LedgerEntries toTestEntries = toTest.read(0, toTest.getLastAddConfirmed()); Iterator toTestIter = toTestEntries.iterator(); From 1e0ee11b685382ef67585c970ca62021dec4e186 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Wed, 31 May 2023 16:11:30 +0800 Subject: [PATCH 144/494] [improve][broker] Do not expose bucketDelayedIndexStats (#20383) (cherry picked from commit 9f93af32d7844003d03cf754ec8cfc50b7177220) --- .../common/policies/data/stats/SubscriptionStatsImpl.java | 2 ++ .../pulsar/common/policies/data/stats/TopicStatsImpl.java | 1 + 2 files changed, 3 insertions(+) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index ea7639a8cd2c6..fb75fecca3363 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.common.policies.data.stats; +import com.fasterxml.jackson.annotation.JsonIgnore; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -134,6 +135,7 @@ public class SubscriptionStatsImpl implements SubscriptionStats { /** The size of InMemoryDelayedDeliveryTracer memory usage. */ public long delayedMessageIndexSizeInBytes; + @JsonIgnore public Map bucketDelayedIndexStats; /** SubscriptionProperties (key/value strings) associated with this subscribe. */ diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java index c9c4739b904f6..ae2438709a017 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java @@ -140,6 +140,7 @@ public class TopicStatsImpl implements TopicStats { public long delayedMessageIndexSizeInBytes; /** Map of bucket delayed index statistics. */ + @JsonIgnore public Map bucketDelayedIndexStats; /** The compaction stats. */ From c0bb12c324c1dc1afd945ac927758154d6025415 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 7 Jun 2023 18:38:18 +0800 Subject: [PATCH 145/494] [fix][ml] There are two same-named managed ledgers in the one broker (#18688) (cherry picked from commit d7186a67fc13ae972a76fbd9ba59b96d8bc7daae) --- .../impl/ManagedLedgerFactoryImpl.java | 16 +++++-- .../mledger/impl/ManagedLedgerImpl.java | 9 ++++ .../impl/ManagedLedgerFactoryTest.java | 43 +++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index 9f3fe9bb0c4a7..cabd70ce4275e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -504,9 +504,19 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob } void close(ManagedLedger ledger) { - // Remove the ledger from the internal factory cache - ledgers.remove(ledger.getName()); - entryCacheManager.removeEntryCache(ledger.getName()); + // If the future in map is not done or has exceptionally complete, it means that @param-ledger is not in the + // map. + CompletableFuture ledgerFuture = ledgers.get(ledger.getName()); + if (ledgerFuture == null || !ledgerFuture.isDone() || ledgerFuture.isCompletedExceptionally()){ + return; + } + if (ledgerFuture.join() != ledger){ + return; + } + // Remove the ledger from the internal factory cache. + if (ledgers.remove(ledger.getName(), ledgerFuture)) { + entryCacheManager.removeEntryCache(ledger.getName()); + } } public CompletableFuture shutdownAsync() throws ManagedLedgerException { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 9b3d7e46aaa8c..10f7948f553cb 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1528,6 +1528,15 @@ private void closeAllCursors(CloseCallback callback, final Object ctx) { @Override public synchronized void createComplete(int rc, final LedgerHandle lh, Object ctx) { + if (STATE_UPDATER.get(this) == State.Closed) { + if (lh != null) { + log.warn("[{}] ledger create completed after the managed ledger is closed rc={} ledger={}, so just" + + " close this ledger handle.", name, rc, lh != null ? lh.getId() : -1); + lh.closeAsync(); + } + return; + } + if (log.isDebugEnabled()) { log.debug("[{}] createComplete rc={} ledger={}", name, rc, lh != null ? lh.getId() : -1); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java index d49d9ab3e2b6b..8695759c99f62 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java @@ -20,12 +20,16 @@ import static org.testng.Assert.assertEquals; +import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.ManagedLedgerInfo.CursorInfo; import org.apache.bookkeeper.mledger.ManagedLedgerInfo.MessageRangeInfo; import org.apache.bookkeeper.test.MockedBookKeeperTestCase; +import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.Test; public class ManagedLedgerFactoryTest extends MockedBookKeeperTestCase { @@ -71,4 +75,43 @@ public void testGetManagedLedgerInfoWithClose() throws Exception { assertEquals(mri.to.entryId, 0); } + /** + * see: https://github.com/apache/pulsar/pull/18688 + */ + @Test + public void testConcurrentCloseLedgerAndSwitchLedgerForReproduceIssue() throws Exception { + String managedLedgerName = "lg_" + UUID.randomUUID().toString().replaceAll("-", "_"); + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setThrottleMarkDelete(1); + config.setMaximumRolloverTime(Integer.MAX_VALUE, TimeUnit.SECONDS); + config.setMaxEntriesPerLedger(5); + + // create managedLedger once and close it. + ManagedLedgerImpl managedLedger1 = (ManagedLedgerImpl) factory.open(managedLedgerName, config); + waitManagedLedgerStateEquals(managedLedger1, ManagedLedgerImpl.State.LedgerOpened); + managedLedger1.close(); + + // create managedLedger the second time. + ManagedLedgerImpl managedLedger2 = (ManagedLedgerImpl) factory.open(managedLedgerName, config); + waitManagedLedgerStateEquals(managedLedger2, ManagedLedgerImpl.State.LedgerOpened); + + // Mock the task create ledger complete now, it will change the state to another value which not is Closed. + // Close managedLedger1 the second time. + managedLedger1.createComplete(1, null, null); + managedLedger1.close(); + + // Verify managedLedger2 is still there. + Assert.assertFalse(factory.ledgers.isEmpty()); + Assert.assertEquals(factory.ledgers.get(managedLedger2.getName()).join(), managedLedger2); + + // cleanup. + managedLedger2.close(); + } + + private void waitManagedLedgerStateEquals(ManagedLedgerImpl managedLedger, ManagedLedgerImpl.State expectedStat){ + Awaitility.await().untilAsserted(() -> + Assert.assertTrue(managedLedger.getState() == expectedStat)); + } + } From 90a82aea99fcacdd72fdead672d6e4787ad24760 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 7 Jun 2023 16:39:45 +0300 Subject: [PATCH 146/494] [fix][broker] Restore solution for certain topic unloading race conditions (#20527) (cherry picked from commit 03f916702ec2a887833543fdea5a2d5305a87302) --- .../java/org/apache/pulsar/broker/service/BrokerService.java | 4 ---- .../pulsar/broker/service/persistent/PersistentTopic.java | 2 +- .../apache/pulsar/broker/service/BrokerBkEnsemblesTests.java | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 33e5500d623d2..4a01fec9abaa7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2218,10 +2218,6 @@ public AuthorizationService getAuthorizationService() { return authorizationService; } - public CompletableFuture removeTopicFromCache(String topicName) { - return removeTopicFutureFromCache(topicName, null); - } - public CompletableFuture removeTopicFromCache(Topic topic) { Optional>> createTopicFuture = findTopicFutureInCache(topic); if (createTopicFuture.isEmpty()){ diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 98e51a2e3ed6e..2a0c229daf6c2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1507,7 +1507,7 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) { } private void disposeTopic(CompletableFuture closeFuture) { - brokerService.removeTopicFromCache(topic) + brokerService.removeTopicFromCache(PersistentTopic.this) .thenRun(() -> { replicatedSubscriptionsController.ifPresent(ReplicatedSubscriptionsController::close); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java index e69714e539be6..9f19bda3647f3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java @@ -127,7 +127,7 @@ public void testCrashBrokerWithoutCursorLedgerLeak() throws Exception { // (3) remove topic and managed-ledger from broker which means topic is not closed gracefully consumer.close(); producer.close(); - pulsar.getBrokerService().removeTopicFromCache(topic1); + pulsar.getBrokerService().removeTopicFromCache(topic); ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory(); Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers"); field.setAccessible(true); @@ -252,7 +252,7 @@ public void testSkipCorruptDataLedger() throws Exception { // clean managed-ledger and recreate topic to clean any data from the cache producer.close(); - pulsar.getBrokerService().removeTopicFromCache(topic1); + pulsar.getBrokerService().removeTopicFromCache(topic); ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory(); Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers"); field.setAccessible(true); From 9385e3878bd4e58b35cb479787d22655cb25a81f Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 7 Jun 2023 20:25:03 +0300 Subject: [PATCH 147/494] [fix][test] Reduce flakiness of AdminApi2Test (#20529) (cherry picked from commit 60dba5d675f998a5108a35be65649edd57ce8596) --- .../pulsar/broker/admin/AdminApi2Test.java | 397 ++++++++++-------- .../broker/testcontext/PulsarTestContext.java | 4 + 2 files changed, 237 insertions(+), 164 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index fb4f880efff07..e1800349bbbb7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.admin; import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.pulsar.broker.BrokerTestUtil.newUniqueName; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -58,7 +59,6 @@ import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.admin.AdminApiTest.MockedPulsarService; @@ -127,6 +127,7 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -135,6 +136,10 @@ public class AdminApi2Test extends MockedPulsarServiceBaseTest { private MockedPulsarService mockPulsarSetup; + private boolean restartClusterAfterTest; + private int usageCount; + private String defaultNamespace; + private String defaultTenant; @BeforeClass @Override @@ -152,10 +157,17 @@ public void setup() throws Exception { @Override protected ServiceConfiguration getDefaultConf() { ServiceConfiguration conf = super.getDefaultConf(); + configureDefaults(conf); + return conf; + } + + void configureDefaults(ServiceConfiguration conf) { conf.setForceDeleteNamespaceAllowed(true); conf.setLoadBalancerEnabled(true); conf.setEnableNamespaceIsolationUpdateOnTime(true); - return conf; + conf.setAllowOverrideEntryFilters(true); + conf.setEntryFilterNames(List.of()); + conf.setMaxNumPartitionsPerPartitionedTopic(0); } @AfterClass(alwaysRun = true) @@ -171,6 +183,20 @@ public void cleanup() throws Exception { @AfterMethod(alwaysRun = true) public void resetClusters() throws Exception { + if (restartClusterAfterTest) { + restartClusterAndResetUsageCount(); + } else { + try { + cleanupCluster(); + } catch (Exception e) { + log.error("Failed to clean up state by deleting namespaces and tenants after test. " + + "Restarting the test broker.", e); + restartClusterAndResetUsageCount(); + } + } + } + + private void cleanupCluster() throws Exception { pulsar.getConfiguration().setForceDeleteTenantAllowed(true); pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); for (String tenant : admin.tenants().getTenants()) { @@ -178,22 +204,59 @@ public void resetClusters() throws Exception { deleteNamespaceWithRetry(namespace, true, admin, pulsar, mockPulsarSetup.getPulsar()); } - admin.tenants().deleteTenant(tenant, true); + try { + admin.tenants().deleteTenant(tenant, true); + } catch (Exception e) { + log.error("Failed to delete tenant {} after test", tenant, e); + String zkDirectory = "/managed-ledgers/" + tenant; + try { + log.info("Listing {} to see if existing keys are preventing deletion.", zkDirectory); + pulsar.getPulsarResources().getLocalMetadataStore().get().getChildren(zkDirectory) + .get(5, TimeUnit.SECONDS).forEach(key -> log.info("Child key '{}'", key)); + } catch (Exception ignore) { + log.error("Failed to list tenant {} ZK directory {} after test", tenant, zkDirectory, e); + } + throw e; + } } for (String cluster : admin.clusters().getClusters()) { admin.clusters().deleteCluster(cluster); } - resetConfig(); + configureDefaults(conf); setupClusters(); } + private void restartClusterAfterTest() { + restartClusterAfterTest = true; + } + + private void restartClusterAndResetUsageCount() throws Exception { + cleanup(); + restartClusterAfterTest = false; + usageCount = 0; + setup(); + } + + private void restartClusterIfReused() throws Exception { + if (usageCount > 1) { + restartClusterAndResetUsageCount(); + } + } + + @BeforeMethod + public void increaseUsageCount() { + usageCount++; + } + private void setupClusters() throws PulsarAdminException { admin.clusters().createCluster("test", ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz", tenantInfo); - admin.namespaces().createNamespace("prop-xyz/ns1", Set.of("test")); + defaultTenant = newUniqueName("prop-xyz"); + admin.tenants().createTenant(defaultTenant, tenantInfo); + defaultNamespace = defaultTenant + "/ns1"; + admin.namespaces().createNamespace(defaultNamespace, Set.of("test")); } @DataProvider(name = "topicType") @@ -250,7 +313,7 @@ public void testIncrementPartitionsOfTopic() throws Exception { final String subName2 = topicName + "-my-sub-2/encode"; final int startPartitions = 4; final int newPartitions = 8; - final String partitionedTopicName = "persistent://prop-xyz/ns1/" + topicName; + final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + topicName; URL pulsarUrl = new URL(pulsar.getWebServiceAddress()); @@ -299,7 +362,7 @@ public void testIncrementPartitionsOfTopic() throws Exception { assertEquals(new HashSet<>(admin.topics().getSubscriptions(newPartitionTopicName)), Set.of(subName1, subName2)); - assertEquals(new HashSet<>(admin.topics().getList("prop-xyz/ns1")).size(), newPartitions); + assertEquals(new HashSet<>(admin.topics().getList(defaultNamespace)).size(), newPartitions); // test cumulative stats for partitioned topic PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false); @@ -333,14 +396,15 @@ public void testIncrementPartitionsOfTopic() throws Exception { @Test public void testTopicPoliciesWithMultiBroker() throws Exception { + restartClusterAfterTest(); + //setup cluster with 3 broker - cleanup(); - setup(); admin.clusters().updateCluster("test", ClusterData.builder().serviceUrl((pulsar.getWebServiceAddress() + ",localhost:1026," + "localhost:2050")).build()); TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz2", tenantInfo); - admin.namespaces().createNamespace("prop-xyz2/ns1", Set.of("test")); + String tenantName = newUniqueName("prop-xyz2"); + admin.tenants().createTenant(tenantName, tenantInfo); + admin.namespaces().createNamespace(tenantName + "/ns1", Set.of("test")); conf.setBrokerServicePort(Optional.of(1024)); conf.setBrokerServicePortTls(Optional.of(1025)); conf.setWebServicePort(Optional.of(1026)); @@ -361,14 +425,14 @@ public void testTopicPoliciesWithMultiBroker() throws Exception { PulsarAdmin admin3 = PulsarAdmin.builder().serviceHttpUrl(pulsar3.getWebServiceAddress()).build(); //for partitioned topic, we can get topic policies from every broker - final String topic = "persistent://prop-xyz2/ns1/" + BrokerTestUtil.newUniqueName("test"); + final String topic = "persistent://" + tenantName + "/ns1/" + newUniqueName("test"); int partitionNum = 3; admin.topics().createPartitionedTopic(topic, partitionNum); pulsarClient.newConsumer().topic(topic).subscriptionName("sub").subscribe().close(); setTopicPoliciesAndValidate(admin2, admin3, topic); //for non-partitioned topic, we can get topic policies from every broker - final String topic2 = "persistent://prop-xyz2/ns1/" + BrokerTestUtil.newUniqueName("test"); + final String topic2 = "persistent://" + tenantName + "/ns1/" + newUniqueName("test"); pulsarClient.newConsumer().topic(topic2).subscriptionName("sub").subscribe().close(); setTopicPoliciesAndValidate(admin2, admin3, topic2); } @@ -402,7 +466,7 @@ private void setTopicPoliciesAndValidate(PulsarAdmin admin2 public void nonPersistentTopics() throws Exception { final String topicName = "nonPersistentTopic"; - final String nonPersistentTopicName = "non-persistent://prop-xyz/ns1/" + topicName; + final String nonPersistentTopicName = "non-persistent://" + defaultNamespace + "/" + topicName; // Force to create a topic publishMessagesOnTopic(nonPersistentTopicName, 0, 0); @@ -434,7 +498,7 @@ public void nonPersistentTopics() throws Exception { assertTrue(topicStats.getSubscriptions().containsKey("my-sub")); assertEquals(topicStats.getPublishers().size(), 0); // test partitioned-topic - final String partitionedTopicName = "non-persistent://prop-xyz/ns1/paritioned"; + final String partitionedTopicName = "non-persistent://" + defaultNamespace + "/paritioned"; admin.topics().createPartitionedTopic(partitionedTopicName, 5); assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 5); } @@ -462,7 +526,7 @@ private void publishMessagesOnTopic(String topicName, int messages, int startIdx @Test public void testSetPersistencePolicies() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); admin.namespaces().createNamespace(namespace, Set.of("test")); assertNull(admin.namespaces().getPersistence(namespace)); @@ -500,7 +564,7 @@ public void testSetPersistencePolicies() throws Exception { @Test public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "persistent://" + namespace + "/topic1"; admin.namespaces().createNamespace(namespace, Set.of("test")); @@ -541,7 +605,7 @@ public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception { @Test(dataProvider = "topicType") public void testUnloadTopic(final String topicType) throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = topicType + "://" + namespace + "/topic1"; admin.namespaces().createNamespace(namespace, Set.of("test")); @@ -591,11 +655,12 @@ private void unloadTopic(String topicName) throws Exception { */ @Test(dataProvider = "namespaceNames", timeOut = 30000) public void testResetCursorOnPosition(String namespaceName) throws Exception { - final String topicName = "persistent://prop-xyz/use/" + namespaceName + "/resetPosition"; + restartClusterAfterTest(); + final String topicName = "persistent://" + defaultTenant + "/use/" + namespaceName + "/resetPosition"; final int totalProducedMessages = 50; // set retention - admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10)); + admin.namespaces().setRetention(defaultNamespace, new RetentionPolicies(10, 10)); // create consumer and subscription Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub") @@ -674,14 +739,11 @@ public void testResetCursorOnPosition(String namespaceName) throws Exception { } consumer.close(); - - cleanup(); - setup(); } @Test public void shouldNotSupportResetOnPartitionedTopic() throws PulsarAdminException, PulsarClientException { - final String partitionedTopicName = "persistent://prop-xyz/ns1/" + BrokerTestUtil.newUniqueName("parttopic"); + final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + newUniqueName("parttopic"); admin.topics().createPartitionedTopic(partitionedTopicName, 4); @Cleanup Consumer consumer = pulsarClient.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub") @@ -713,7 +775,9 @@ private void publishMessagesOnPersistentTopic(String topicName, int messages, in @Test(timeOut = 20000) public void testMaxConsumersOnSubApi() throws Exception { - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); + assertNull(admin.namespaces().getMaxConsumersPerSubscription(namespace)); admin.namespaces().setMaxConsumersPerSubscription(namespace, 10); Awaitility.await().untilAsserted(() -> { @@ -732,7 +796,7 @@ public void testMaxConsumersOnSubApi() throws Exception { */ @Test public void testLoadReportApi() throws Exception { - + restartClusterAfterTest(); this.conf.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); @Cleanup("cleanup") MockedPulsarService mockPulsarSetup1 = new MockedPulsarService(this.conf); @@ -795,6 +859,8 @@ public void testPeerCluster() throws Exception { */ @Test public void testReplicationPeerCluster() throws Exception { + restartClusterAfterTest(); + admin.clusters().createCluster("us-west1", ClusterData.builder().serviceUrl("http://broker.messaging.west1.example.com:8080").build()); admin.clusters().createCluster("us-west2", @@ -814,7 +880,7 @@ public void testReplicationPeerCluster() throws Exception { assertEquals(allClusters, List.of("test", "us-east1", "us-east2", "us-west1", "us-west2", "us-west3", "us-west4")); - final String property = "peer-prop"; + final String property = newUniqueName("peer-prop"); Set allowedClusters = Set.of("us-west1", "us-west2", "us-west3", "us-west4", "us-east1", "us-east2", "global"); TenantInfoImpl propConfig = new TenantInfoImpl(Set.of("test"), allowedClusters); @@ -848,9 +914,6 @@ public void testReplicationPeerCluster() throws Exception { clusterIds = Set.of("us-west1", "us-west4"); // no peer coexist in replication clusters admin.namespaces().setNamespaceReplicationClusters(namespace, clusterIds); - - cleanup(); - setup(); } @Test @@ -889,7 +952,7 @@ public void clusterFailureDomain() throws PulsarAdminException { @Test public void namespaceAntiAffinity() throws PulsarAdminException { - final String namespace = "prop-xyz/ns1"; + final String namespace = defaultNamespace; final String antiAffinityGroup = "group"; assertTrue(isBlank(admin.namespaces().getNamespaceAntiAffinityGroup(namespace))); admin.namespaces().setNamespaceAntiAffinityGroup(namespace, antiAffinityGroup); @@ -897,9 +960,9 @@ public void namespaceAntiAffinity() throws PulsarAdminException { admin.namespaces().deleteNamespaceAntiAffinityGroup(namespace); assertTrue(isBlank(admin.namespaces().getNamespaceAntiAffinityGroup(namespace))); - final String ns1 = "prop-xyz/antiAG1"; - final String ns2 = "prop-xyz/antiAG2"; - final String ns3 = "prop-xyz/antiAG3"; + final String ns1 = defaultTenant + "/antiAG1"; + final String ns2 = defaultTenant + "/antiAG2"; + final String ns3 = defaultTenant + "/antiAG3"; admin.namespaces().createNamespace(ns1, Set.of("test")); admin.namespaces().createNamespace(ns2, Set.of("test")); admin.namespaces().createNamespace(ns3, Set.of("test")); @@ -908,19 +971,19 @@ public void namespaceAntiAffinity() throws PulsarAdminException { admin.namespaces().setNamespaceAntiAffinityGroup(ns3, antiAffinityGroup); Set namespaces = new HashSet<>( - admin.namespaces().getAntiAffinityNamespaces("prop-xyz", "test", antiAffinityGroup)); + admin.namespaces().getAntiAffinityNamespaces(defaultTenant, "test", antiAffinityGroup)); assertEquals(namespaces.size(), 3); assertTrue(namespaces.contains(ns1)); assertTrue(namespaces.contains(ns2)); assertTrue(namespaces.contains(ns3)); - List namespaces2 = admin.namespaces().getAntiAffinityNamespaces("prop-xyz", "test", "invalid-group"); + List namespaces2 = admin.namespaces().getAntiAffinityNamespaces(defaultTenant, "test", "invalid-group"); assertEquals(namespaces2.size(), 0); } @Test public void testPersistentTopicList() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "non-persistent://" + namespace + "/bundle-topic"; admin.namespaces().createNamespace(namespace, 20); admin.namespaces().setNamespaceReplicationClusters(namespace, Set.of("test")); @@ -954,7 +1017,7 @@ public void testPersistentTopicList() throws Exception { @Test public void testCreateAndGetTopicProperties() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String nonPartitionedTopicName = "persistent://" + namespace + "/non-partitioned-TopicProperties"; admin.namespaces().createNamespace(namespace, 20); Map nonPartitionedTopicProperties = new HashMap<>(); @@ -975,7 +1038,7 @@ public void testCreateAndGetTopicProperties() throws Exception { @Test public void testUpdatePartitionedTopicProperties() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "persistent://" + namespace + "/testUpdatePartitionedTopicProperties"; final String topicNameTwo = "persistent://" + namespace + "/testUpdatePartitionedTopicProperties2"; admin.namespaces().createNamespace(namespace, 20); @@ -1033,7 +1096,7 @@ public void testUpdatePartitionedTopicProperties() throws Exception { @Test public void testUpdateNonPartitionedTopicProperties() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "persistent://" + namespace + "/testUpdateNonPartitionedTopicProperties"; admin.namespaces().createNamespace(namespace, 20); @@ -1068,7 +1131,7 @@ public void testUpdateNonPartitionedTopicProperties() throws Exception { @Test public void testNonPersistentTopics() throws Exception { - final String namespace = "prop-xyz/ns2"; + final String namespace = newUniqueName(defaultTenant + "/ns2"); final String topicName = "non-persistent://" + namespace + "/topic"; admin.namespaces().createNamespace(namespace, 20); admin.namespaces().setNamespaceReplicationClusters(namespace, Set.of("test")); @@ -1096,7 +1159,7 @@ public void testNonPersistentTopics() throws Exception { public void testPublishConsumerStats() throws Exception { final String topicName = "statTopic"; final String subscriberName = topicName + "-my-sub-1"; - final String topic = "persistent://prop-xyz/ns1/" + topicName; + final String topic = "persistent://" + defaultNamespace + "/" + topicName; final String producerName = "myProducer"; @Cleanup @@ -1143,6 +1206,8 @@ public void testPublishConsumerStats() throws Exception { @Test public void testTenantNameWithUnderscore() throws Exception { + restartClusterAfterTest(); + TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); admin.tenants().createTenant("prop_xyz", tenantInfo); @@ -1150,15 +1215,12 @@ public void testTenantNameWithUnderscore() throws Exception { String topic = "persistent://prop_xyz/use/my-namespace/my-topic"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) .create(); TopicStats stats = admin.topics().getStats(topic); assertEquals(stats.getPublishers().size(), 1); - producer.close(); - - cleanup(); - setup(); } @Test @@ -1200,11 +1262,11 @@ public void testTenantWithNonexistentClusters() throws Exception { assertFalse(admin.tenants().getTenants().contains("test-tenant")); // Check existing tenant - assertTrue(admin.tenants().getTenants().contains("prop-xyz")); + assertTrue(admin.tenants().getTenants().contains(defaultTenant)); // If we try to update existing tenant with nonexistent clusters, it should fail immediately try { - admin.tenants().updateTenant("prop-xyz", tenantInfo); + admin.tenants().updateTenant(defaultTenant, tenantInfo); fail("Should have failed"); } catch (PulsarAdminException e) { assertEquals(e.getStatusCode(), Status.PRECONDITION_FAILED.getStatusCode()); @@ -1264,7 +1326,7 @@ public void brokerNamespaceIsolationPolicies() throws Exception { @Test public void brokerNamespaceIsolationPoliciesUpdateOnTime() throws Exception { String brokerName = pulsar.getAdvertisedAddress(); - String ns1Name = "prop-xyz/test_ns1_iso_" + System.currentTimeMillis(); + String ns1Name = defaultTenant + "/test_ns1_iso_" + System.currentTimeMillis(); admin.namespaces().createNamespace(ns1Name, Set.of("test")); // 0. without isolation policy configured, lookup will success. @@ -1335,15 +1397,16 @@ public void clustersList() throws PulsarAdminException { */ @Test public void testClusterIsReadyBeforeCreateTopic() throws Exception { + restartClusterAfterTest(); final String topicName = "partitionedTopic"; final int partitions = 4; - final String persistentPartitionedTopicName = "persistent://prop-xyz/ns2/" + topicName; - final String NonPersistentPartitionedTopicName = "non-persistent://prop-xyz/ns2/" + topicName; + final String persistentPartitionedTopicName = "persistent://" + defaultTenant + "/ns2/" + topicName; + final String NonPersistentPartitionedTopicName = "non-persistent://" + defaultTenant + "/ns2/" + topicName; - admin.namespaces().createNamespace("prop-xyz/ns2"); + admin.namespaces().createNamespace(defaultTenant + "/ns2"); // By default the cluster will configure as configuration file. So the create topic operation // will never throw exception except there is no cluster. - admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns2", new HashSet()); + admin.namespaces().setNamespaceReplicationClusters(defaultTenant + "/ns2", new HashSet()); try { admin.topics().createPartitionedTopic(persistentPartitionedTopicName, partitions); @@ -1356,15 +1419,12 @@ public void testClusterIsReadyBeforeCreateTopic() throws Exception { Assert.fail("should have failed due to Namespace does not have any clusters configured"); } catch (PulsarAdminException.PreconditionFailedException ignored) { } - - cleanup(); - setup(); } @Test public void testCreateNamespaceWithNoClusters() throws PulsarAdminException { String localCluster = pulsar.getConfiguration().getClusterName(); - String namespace = "prop-xyz/test-ns-with-no-clusters"; + String namespace = newUniqueName(defaultTenant + "/test-ns-with-no-clusters"); admin.namespaces().createNamespace(namespace); // Global cluster, if there, should be omitted from the results @@ -1377,7 +1437,7 @@ public void testConsumerStatsLastTimestamp() throws PulsarClientException, Pulsa long timestamp = System.currentTimeMillis(); final String topicName = "consumer-stats-" + timestamp; final String subscribeName = topicName + "-test-stats-sub"; - final String topic = "persistent://prop-xyz/ns1/" + topicName; + final String topic = "persistent://" + defaultNamespace + "/" + topicName; final String producerName = "producer-" + topicName; @Cleanup @@ -1480,10 +1540,9 @@ public void testConsumerStatsLastTimestamp() throws PulsarClientException, Pulsa @Test(timeOut = 30000) public void testPreciseBacklog() throws Exception { - cleanup(); - setup(); + restartClusterIfReused(); - final String topic = "persistent://prop-xyz/ns1/precise-back-log"; + final String topic = "persistent://" + defaultNamespace + "/precise-back-log"; final String subName = "sub-name"; @Cleanup @@ -1533,6 +1592,7 @@ public void testPreciseBacklog() throws Exception { @Test public void testDeleteTenant() throws Exception { + restartClusterAfterTest(); // Disabled conf: systemTopicEnabled. see: https://github.com/apache/pulsar/pull/17070 boolean originalSystemTopicEnabled = conf.isSystemTopicEnabled(); if (originalSystemTopicEnabled) { @@ -1542,7 +1602,7 @@ public void testDeleteTenant() throws Exception { } pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); - String tenant = "test-tenant-1"; + String tenant = newUniqueName("test-tenant-1"); assertFalse(admin.tenants().getTenants().contains(tenant)); // create tenant @@ -1588,12 +1648,6 @@ public void testDeleteTenant() throws Exception { assertFalse(pulsar.getLocalMetadataStore().exists(partitionedTopicPath).join()); assertFalse(pulsar.getLocalMetadataStore().exists(localPoliciesPath).join()); assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); - // Reset conf: systemTopicEnabled - if (originalSystemTopicEnabled) { - cleanup(); - conf.setSystemTopicEnabled(true); - setup(); - } } @Data @@ -1628,13 +1682,14 @@ private void setNamespaceAttr(NamespaceAttr namespaceAttr){ @Test(dataProvider = "namespaceAttrs") public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { + restartClusterAfterTest(); + // Set conf. cleanup(); - NamespaceAttr originalNamespaceAttr = markOriginalNamespaceAttr(); setNamespaceAttr(namespaceAttr); setup(); - String tenant = "test-tenant"; + String tenant = newUniqueName("test-tenant"); assertFalse(admin.tenants().getTenants().contains(tenant)); // create tenant @@ -1675,16 +1730,11 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { final String bundleDataPath = "/loadbalance/bundle-data/" + namespace; assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); - - // Reset config - cleanup(); - setNamespaceAttr(originalNamespaceAttr); - setup(); } @Test public void testDeleteNamespaceWithTopicPolicies() throws Exception { - String tenant = "test-tenant"; + String tenant = newUniqueName("test-tenant"); assertFalse(admin.tenants().getTenants().contains(tenant)); // create tenant @@ -1732,7 +1782,7 @@ public void testDeleteNamespaceWithTopicPolicies() throws Exception { @Test(timeOut = 30000) public void testBacklogNoDelayed() throws PulsarClientException, PulsarAdminException, InterruptedException { - final String topic = "persistent://prop-xyz/ns1/precise-back-log-no-delayed-" + UUID.randomUUID().toString(); + final String topic = "persistent://" + defaultNamespace + "/precise-back-log-no-delayed-" + UUID.randomUUID().toString(); final String subName = "sub-name"; @Cleanup @@ -1788,7 +1838,7 @@ public void testBacklogNoDelayed() throws PulsarClientException, PulsarAdminExce @Test public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException, PulsarAdminException { - final String topic = "persistent://prop-xyz/ns1/precise-back-log-for-partitioned-topic"; + final String topic = "persistent://" + defaultNamespace + "/precise-back-log-for-partitioned-topic"; admin.topics().createPartitionedTopic(topic, 2); final String subName = "sub-name"; @@ -1830,7 +1880,7 @@ public void testPreciseBacklogForPartitionedTopic() throws PulsarClientException @Test(timeOut = 30000) public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientException, PulsarAdminException, InterruptedException { - final String topic = "persistent://prop-xyz/ns1/precise-back-log-no-delayed-partitioned-topic"; + final String topic = "persistent://" + defaultNamespace + "/precise-back-log-no-delayed-partitioned-topic"; admin.topics().createPartitionedTopic(topic, 2); final String subName = "sub-name"; @@ -1884,7 +1934,8 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti @Test public void testMaxNumPartitionsPerPartitionedTopicSuccess() { - final String topic = "persistent://prop-xyz/ns1/max-num-partitions-per-partitioned-topic-success"; + restartClusterAfterTest(); + final String topic = "persistent://" + defaultNamespace + "/max-num-partitions-per-partitioned-topic-success"; pulsar.getConfiguration().setMaxNumPartitionsPerPartitionedTopic(3); try { @@ -1899,7 +1950,8 @@ public void testMaxNumPartitionsPerPartitionedTopicSuccess() { @Test public void testMaxNumPartitionsPerPartitionedTopicFailure() { - final String topic = "persistent://prop-xyz/ns1/max-num-partitions-per-partitioned-topic-failure"; + restartClusterAfterTest(); + final String topic = "persistent://" + defaultNamespace + "/max-num-partitions-per-partitioned-topic-failure"; pulsar.getConfiguration().setMaxNumPartitionsPerPartitionedTopic(2); try { @@ -1916,21 +1968,24 @@ public void testMaxNumPartitionsPerPartitionedTopicFailure() { @Test public void testListOfNamespaceBundles() throws Exception { TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz2", tenantInfo); - admin.namespaces().createNamespace("prop-xyz2/ns1", 10); - admin.namespaces().setNamespaceReplicationClusters("prop-xyz2/ns1", Set.of("test")); - admin.namespaces().createNamespace("prop-xyz2/test/ns2", 10); - assertEquals(admin.namespaces().getBundles("prop-xyz2/ns1").getNumBundles(), 10); - assertEquals(admin.namespaces().getBundles("prop-xyz2/test/ns2").getNumBundles(), 10); + String tenantName = newUniqueName("prop-xyz2"); + admin.tenants().createTenant(tenantName, tenantInfo); + admin.namespaces().createNamespace(tenantName + "/ns1", 10); + admin.namespaces().setNamespaceReplicationClusters(tenantName + "/ns1", Set.of("test")); + admin.namespaces().createNamespace(tenantName + "/test/ns2", 10); + assertEquals(admin.namespaces().getBundles(tenantName + "/ns1").getNumBundles(), 10); + assertEquals(admin.namespaces().getBundles(tenantName + "/test/ns2").getNumBundles(), 10); - admin.namespaces().deleteNamespace("prop-xyz2/test/ns2"); + admin.namespaces().deleteNamespace(tenantName + "/test/ns2"); } @Test public void testForceDeleteNamespace() throws Exception { - final String namespaceName = "prop-xyz2/ns1"; + restartClusterAfterTest(); + String tenantName = newUniqueName("prop-xyz2"); + final String namespaceName = tenantName + "/ns1"; TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz2", tenantInfo); + admin.tenants().createTenant(tenantName, tenantInfo); admin.namespaces().createNamespace(namespaceName, 1); final String topic = "persistent://" + namespaceName + "/test" + UUID.randomUUID(); pulsarClient.newProducer(Schema.DOUBLE).topic(topic).create().close(); @@ -1942,17 +1997,15 @@ public void testForceDeleteNamespace() throws Exception { } catch (PulsarAdminException e) { assertEquals(e.getStatusCode(), 404); } - - cleanup(); - setup(); } @Test public void testForceDeleteNamespaceWithAutomaticTopicCreation() throws Exception { conf.setForceDeleteNamespaceAllowed(true); - final String namespaceName = "prop-xyz2/ns1"; + String tenantName = newUniqueName("prop-xyz2"); + final String namespaceName = tenantName + "/ns1"; TenantInfoImpl tenantInfo = new TenantInfoImpl(Set.of("role1", "role2"), Set.of("test")); - admin.tenants().createTenant("prop-xyz2", tenantInfo); + admin.tenants().createTenant(tenantName, tenantInfo); admin.namespaces().createNamespace(namespaceName, 1); admin.namespaces().setAutoTopicCreation(namespaceName, AutoTopicCreationOverride.builder() @@ -1977,7 +2030,7 @@ public void testForceDeleteNamespaceWithAutomaticTopicCreation() throws Exceptio // the consumer will try to re-create the partitions admin.namespaces().deleteNamespace(namespaceName, true); - assertFalse(admin.namespaces().getNamespaces("prop-xyz2").contains("ns1")); + assertFalse(admin.namespaces().getNamespaces(tenantName).contains("ns1")); } } @@ -2000,6 +2053,7 @@ public void testUpdateClusterWithProxyUrl() throws Exception { @Test public void testMaxNamespacesPerTenant() throws Exception { + restartClusterAfterTest(); cleanup(); conf.setMaxNamespacesPerTenant(2); setup(); @@ -2013,19 +2067,11 @@ public void testMaxNamespacesPerTenant() throws Exception { Assert.assertEquals(e.getStatusCode(), 412); Assert.assertEquals(e.getHttpError(), "Exceed the maximum number of namespace in tenant :testTenant"); } - - //unlimited - cleanup(); - conf.setMaxNamespacesPerTenant(0); - setup(); - admin.tenants().createTenant("testTenant", tenantInfo); - for (int i = 0; i < 10; i++) { - admin.namespaces().createNamespace("testTenant/ns-" + i, Set.of("test")); - } } @Test public void testAutoTopicCreationOverrideWithMaxNumPartitionsLimit() throws Exception{ + restartClusterAfterTest(); cleanup(); conf.setMaxNumPartitionsPerPartitionedTopic(10); setup(); @@ -2067,6 +2113,7 @@ public void testAutoTopicCreationOverrideWithMaxNumPartitionsLimit() throws Exce } @Test public void testMaxTopicsPerNamespace() throws Exception { + restartClusterAfterTest(); cleanup(); conf.setMaxTopicsPerNamespace(10); setup(); @@ -2158,16 +2205,12 @@ public void testMaxTopicsPerNamespace() throws Exception { } catch (PulsarClientException e) { log.info("Exception: ", e); } - - // reset configuration - conf.setMaxTopicsPerNamespace(0); - conf.setDefaultNumPartitions(1); } @Test public void testInvalidBundleErrorResponse() throws Exception { try { - admin.namespaces().deleteNamespaceBundle("prop-xyz/ns1", "invalid-bundle"); + admin.namespaces().deleteNamespaceBundle(defaultNamespace, "invalid-bundle"); fail("should have failed due to invalid bundle"); } catch (PreconditionFailedException e) { assertTrue(e.getMessage().startsWith("Invalid bundle range")); @@ -2176,6 +2219,7 @@ public void testInvalidBundleErrorResponse() throws Exception { @Test public void testMaxSubscriptionsPerTopic() throws Exception { + restartClusterAfterTest(); cleanup(); conf.setMaxSubscriptionsPerTopic(2); setup(); @@ -2258,7 +2302,7 @@ public void testMaxSubscriptionsPerTopic() throws Exception { @Test(timeOut = 30000) public void testMaxSubPerTopicApi() throws Exception { - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); assertNull(admin.namespaces().getMaxSubscriptionsPerTopic(myNamespace)); @@ -2281,8 +2325,11 @@ public void testMaxSubPerTopicApi() throws Exception { } } - @Test(timeOut = 30000) + @Test(timeOut = 60000) public void testSetNamespaceEntryFilters() throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); @@ -2299,7 +2346,7 @@ public void testSetNamespaceEntryFilters() throws Exception { try { EntryFilters entryFilters = new EntryFilters("test"); - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); final String topicName = myNamespace + "/topic"; admin.topics().createNonPartitionedTopic(topicName); @@ -2359,6 +2406,9 @@ public void testSetNamespaceEntryFilters() throws Exception { @Test(dataProvider = "topicType") public void testSetTopicLevelEntryFilters(String topicType) throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); @@ -2373,7 +2423,7 @@ public void testSetTopicLevelEntryFilters(String topicType) throws Exception { "entryFilterProvider", testEntryFilterProvider, true); try { EntryFilters entryFilters = new EntryFilters("test"); - final String topic = topicType + "://prop-xyz/ns1/test-schema-validation-enforced"; + final String topic = topicType + "://" + defaultNamespace + "/test-schema-validation-enforced"; admin.topics().createPartitionedTopic(topic, 1); final String fullTopicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + 0; @Cleanup @@ -2431,13 +2481,13 @@ public void testSetTopicLevelEntryFilters(String topicType) throws Exception { } } - @Test(timeOut = 30000) + @Test(timeOut = 60000) public void testSetEntryFiltersHierarchy() throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); - conf.setEntryFilterNames(List.of("test", "test1")); - conf.setAllowOverrideEntryFilters(true); - testEntryFilterProvider.setMockEntryFilters(new EntryFilterDefinition( "test", null, @@ -2450,9 +2500,11 @@ public void testSetEntryFiltersHierarchy() throws Exception { final EntryFilterProvider oldEntryFilterProvider = pulsar.getBrokerService().getEntryFilterProvider(); FieldUtils.writeField(pulsar.getBrokerService(), "entryFilterProvider", testEntryFilterProvider, true); + conf.setEntryFilterNames(List.of("test", "test1")); + conf.setAllowOverrideEntryFilters(true); try { - final String topic = "persistent://prop-xyz/ns1/test-schema-validation-enforced"; + final String topic = "persistent://" + defaultNamespace + "/test-schema-validation-enforced"; admin.topics().createPartitionedTopic(topic, 1); final String fullTopicName = topic + TopicName.PARTITIONED_TOPIC_SUFFIX + 0; @Cleanup @@ -2460,8 +2512,9 @@ public void testSetEntryFiltersHierarchy() throws Exception { .topic(fullTopicName) .create(); assertNull(admin.topicPolicies().getEntryFiltersPerTopic(topic, false)); - assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), - new EntryFilters("test,test1")); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), + new EntryFilters("test,test1"))); assertEquals(pulsar .getBrokerService() .getTopic(fullTopicName, false) @@ -2471,10 +2524,11 @@ public void testSetEntryFiltersHierarchy() throws Exception { .size(), 2); EntryFilters nsEntryFilters = new EntryFilters("test"); - admin.namespaces().setNamespaceEntryFilters("prop-xyz/ns1", nsEntryFilters); - assertEquals(admin.namespaces().getNamespaceEntryFilters("prop-xyz/ns1"), nsEntryFilters); - assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), - new EntryFilters("test")); + admin.namespaces().setNamespaceEntryFilters(defaultNamespace, nsEntryFilters); + assertEquals(admin.namespaces().getNamespaceEntryFilters(defaultNamespace), nsEntryFilters); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), + new EntryFilters("test"))); Awaitility.await().untilAsserted(() -> { assertEquals(pulsar .getBrokerService() @@ -2503,8 +2557,9 @@ public void testSetEntryFiltersHierarchy() throws Exception { admin.topicPolicies().setEntryFiltersPerTopic(topic, topicEntryFilters); Awaitility.await().untilAsserted(() -> assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, false), topicEntryFilters)); - assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), - new EntryFilters("test1")); + Awaitility.await().untilAsserted(() -> + assertEquals(admin.topicPolicies().getEntryFiltersPerTopic(topic, true), + new EntryFilters("test1"))); Awaitility.await().untilAsserted(() -> { assertEquals(pulsar .getBrokerService() @@ -2530,8 +2585,11 @@ public void testSetEntryFiltersHierarchy() throws Exception { } } - @Test(timeOut = 30000) + @Test(timeOut = 60000) public void testValidateNamespaceEntryFilters() throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); @@ -2546,7 +2604,7 @@ public void testValidateNamespaceEntryFilters() throws Exception { "entryFilterProvider", testEntryFilterProvider, true); try { - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); try { admin.namespaces().setNamespaceEntryFilters(myNamespace, new EntryFilters("notexists")); @@ -2585,8 +2643,11 @@ public void testValidateNamespaceEntryFilters() throws Exception { } } - @Test(timeOut = 30000) + @Test(timeOut = 60000) public void testValidateTopicEntryFilters() throws Exception { + restartClusterAfterTest(); + restartClusterIfReused(); + @Cleanup final MockEntryFilterProvider testEntryFilterProvider = new MockEntryFilterProvider(conf); @@ -2601,7 +2662,7 @@ public void testValidateTopicEntryFilters() throws Exception { "entryFilterProvider", testEntryFilterProvider, true); try { - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); final String topicName = myNamespace + "/topic"; admin.topics().createNonPartitionedTopic(topicName); @@ -2648,8 +2709,9 @@ public void testValidateTopicEntryFilters() throws Exception { @Test(timeOut = 30000) public void testMaxSubPerTopic() throws Exception { + restartClusterAfterTest(); pulsar.getConfiguration().setMaxSubscriptionsPerTopic(0); - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); final String topic = "persistent://" + myNamespace + "/testMaxSubPerTopic"; pulsarClient.newProducer().topic(topic).create().close(); @@ -2689,12 +2751,13 @@ public void testMaxSubPerTopic() throws Exception { @Test(timeOut = 30000) public void testMaxSubPerTopicPriority() throws Exception { + restartClusterAfterTest(); final int brokerLevelMaxSub = 2; cleanup(); conf.setMaxSubscriptionsPerTopic(brokerLevelMaxSub); setup(); - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); final String topic = "persistent://" + myNamespace + "/testMaxSubPerTopic"; //Create a client that can fail quickly @@ -2746,12 +2809,13 @@ public void testMaxSubPerTopicPriority() throws Exception { @Test public void testMaxProducersPerTopicUnlimited() throws Exception { + restartClusterAfterTest(); final int maxProducersPerTopic = 1; cleanup(); conf.setMaxProducersPerTopic(maxProducersPerTopic); setup(); //init namespace - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); final String topic = "persistent://" + myNamespace + "/testMaxProducersPerTopicUnlimited"; //the policy is set to 0, so there will be no restrictions @@ -2797,12 +2861,13 @@ public void testMaxProducersPerTopicUnlimited() throws Exception { @Test public void testMaxConsumersPerTopicUnlimited() throws Exception { + restartClusterAfterTest(); final int maxConsumersPerTopic = 1; cleanup(); conf.setMaxConsumersPerTopic(maxConsumersPerTopic); setup(); //init namespace - final String myNamespace = "prop-xyz/ns" + UUID.randomUUID(); + final String myNamespace = newUniqueName(defaultTenant + "/ns"); admin.namespaces().createNamespace(myNamespace, Set.of("test")); final String topic = "persistent://" + myNamespace + "/testMaxConsumersPerTopicUnlimited"; @@ -2854,7 +2919,7 @@ public void testMaxConsumersPerTopicUnlimited() throws Exception { @Test public void testClearBacklogForTheSubscriptionThatNoConsumers() throws Exception { - final String topic = "persistent://prop-xyz/ns1/clear_backlog_no_consumers" + UUID.randomUUID().toString(); + final String topic = "persistent://" + defaultNamespace + "/clear_backlog_no_consumers" + UUID.randomUUID().toString(); final String sub = "my-sub"; admin.topics().createNonPartitionedTopic(topic); admin.topics().createSubscription(topic, sub, MessageId.earliest); @@ -2863,7 +2928,10 @@ public void testClearBacklogForTheSubscriptionThatNoConsumers() throws Exception @Test(timeOut = 200000) public void testCompactionApi() throws Exception { - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); + + assertNull(admin.namespaces().getCompactionThreshold(namespace)); assertEquals(pulsar.getConfiguration().getBrokerServiceCompactionThresholdInBytes(), 0); @@ -2878,11 +2946,13 @@ public void testCompactionApi() throws Exception { @Test(timeOut = 200000) public void testCompactionPriority() throws Exception { + restartClusterAfterTest(); cleanup(); conf.setBrokerServiceCompactionMonitorIntervalInSeconds(10000); setup(); - final String topic = "persistent://prop-xyz/ns1/topic" + UUID.randomUUID(); - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); + final String topic = "persistent://" + namespace + "/topic" + UUID.randomUUID(); pulsarClient.newProducer().topic(topic).create().close(); TopicName topicName = TopicName.get(topic); PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get().get(); @@ -2921,7 +2991,8 @@ public void testCompactionPriority() throws Exception { @Test public void testProperties() throws Exception { - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); admin.namespaces().setProperty(namespace, "a", "a"); assertEquals("a", admin.namespaces().getProperty(namespace, "a")); assertNull(admin.namespaces().getProperty(namespace, "b")); @@ -2944,7 +3015,7 @@ public void testProperties() throws Exception { @Test public void testGetListInBundle() throws Exception { - final String namespace = "prop-xyz/ns11"; + final String namespace = defaultTenant + "/ns11"; admin.namespaces().createNamespace(namespace, 3); final String persistentTopicName = TopicName.get( @@ -2978,7 +3049,8 @@ public void testGetListInBundle() throws Exception { @Test public void testGetTopicsWithDifferentMode() throws Exception { - final String namespace = "prop-xyz/ns1"; + final String namespace = newUniqueName(defaultTenant + "/ns"); + admin.namespaces().createNamespace(namespace, Set.of("test")); final String persistentTopicName = TopicName .get("persistent", NamespaceName.get(namespace), "get_topics_mode_" + UUID.randomUUID().toString()) @@ -3021,16 +3093,14 @@ public void testGetTopicsWithDifferentMode() throws Exception { @Test(dataProvider = "isV1") public void testNonPartitionedTopic(boolean isV1) throws Exception { - String tenant = "prop-xyz"; + restartClusterAfterTest(); + String tenant = defaultTenant; String cluster = "test"; String namespace = tenant + "/" + (isV1 ? cluster + "/" : "") + "n1" + isV1; String topic = "persistent://" + namespace + "/t1" + isV1; admin.namespaces().createNamespace(namespace, Set.of(cluster)); admin.topics().createNonPartitionedTopic(topic); assertTrue(admin.topics().getList(namespace).contains(topic)); - - cleanup(); - setup(); } /** @@ -3043,7 +3113,7 @@ public void testFailedUpdatePartitionedTopic() throws Exception { final String subName1 = topicName + "-my-sub-1"; final int startPartitions = 4; final int newPartitions = 8; - final String partitionedTopicName = "persistent://prop-xyz/ns1/" + topicName; + final String partitionedTopicName = "persistent://" + defaultNamespace + "/" + topicName; URL pulsarUrl = new URL(pulsar.getWebServiceAddress()); @@ -3079,11 +3149,9 @@ public void testFailedUpdatePartitionedTopic() throws Exception { @Test(dataProvider = "topicType") public void testPartitionedStatsAggregationByProducerName(String topicType) throws Exception { - cleanup(); - setup(); - + restartClusterIfReused(); conf.setAggregatePublisherStatsByProducerName(true); - final String topic = topicType + "://prop-xyz/ns1/test-partitioned-stats-aggregation-by-producer-name"; + final String topic = topicType + "://" + defaultNamespace + "/test-partitioned-stats-aggregation-by-producer-name"; admin.topics().createPartitionedTopic(topic, 10); @Cleanup @@ -3137,8 +3205,9 @@ public int choosePartition(Message msg, TopicMetadata metadata) { @Test(dataProvider = "topicType") public void testPartitionedStatsAggregationByProducerNamePerPartition(String topicType) throws Exception { + restartClusterIfReused(); conf.setAggregatePublisherStatsByProducerName(true); - final String topic = topicType + "://prop-xyz/ns1/test-partitioned-stats-aggregation-by-producer-name-per-pt"; + final String topic = topicType + "://" + defaultNamespace + "/test-partitioned-stats-aggregation-by-producer-name-per-pt"; admin.topics().createPartitionedTopic(topic, 2); @Cleanup @@ -3161,7 +3230,7 @@ public void testPartitionedStatsAggregationByProducerNamePerPartition(String top @Test(dataProvider = "topicType") public void testSchemaValidationEnforced(String topicType) throws Exception { - final String topic = topicType + "://prop-xyz/ns1/test-schema-validation-enforced"; + final String topic = topicType + "://" + defaultNamespace + "/test-schema-validation-enforced"; admin.topics().createPartitionedTopic(topic, 1); @Cleanup Producer producer1 = pulsarClient.newProducer() @@ -3177,31 +3246,31 @@ public void testSchemaValidationEnforced(String topicType) throws Exception { @Test public void testGetNamespaceTopicList() throws Exception { - final String persistentTopic = "persistent://prop-xyz/ns1/testGetNamespaceTopicList"; - final String nonPersistentTopic = "non-persistent://prop-xyz/ns1/non-testGetNamespaceTopicList"; - final String eventTopic = "persistent://prop-xyz/ns1/__change_events"; + final String persistentTopic = "persistent://" + defaultNamespace + "/testGetNamespaceTopicList"; + final String nonPersistentTopic = "non-persistent://" + defaultNamespace + "/non-testGetNamespaceTopicList"; + final String eventTopic = "persistent://" + defaultNamespace + "/__change_events"; admin.topics().createNonPartitionedTopic(persistentTopic); Awaitility.await().untilAsserted(() -> - admin.namespaces().getTopics("prop-xyz/ns1", + admin.namespaces().getTopics(defaultNamespace, ListNamespaceTopicsOptions.builder().mode(Mode.PERSISTENT).includeSystemTopic(true).build()) .contains(eventTopic)); - List notIncludeSystemTopics = admin.namespaces().getTopics("prop-xyz/ns1", + List notIncludeSystemTopics = admin.namespaces().getTopics(defaultNamespace, ListNamespaceTopicsOptions.builder().includeSystemTopic(false).build()); Assert.assertFalse(notIncludeSystemTopics.contains(eventTopic)); @Cleanup Producer producer = pulsarClient.newProducer() .topic(nonPersistentTopic) .create(); - List notPersistentTopics = admin.namespaces().getTopics("prop-xyz/ns1", + List notPersistentTopics = admin.namespaces().getTopics(defaultNamespace, ListNamespaceTopicsOptions.builder().mode(Mode.NON_PERSISTENT).build()); Assert.assertTrue(notPersistentTopics.contains(nonPersistentTopic)); } @Test private void testTerminateSystemTopic() throws Exception { - final String topic = "persistent://prop-xyz/ns1/testTerminateSystemTopic"; + final String topic = "persistent://" + defaultNamespace + "/testTerminateSystemTopic"; admin.topics().createNonPartitionedTopic(topic); - final String eventTopic = "persistent://prop-xyz/ns1/__change_events"; + final String eventTopic = "persistent://" + defaultNamespace + "/__change_events"; admin.topicPolicies().setMaxConsumers(topic, 2); Awaitility.await().untilAsserted(() -> { Assert.assertEquals(admin.topicPolicies().getMaxConsumers(topic), Integer.valueOf(2)); @@ -3213,12 +3282,12 @@ private void testTerminateSystemTopic() throws Exception { @Test private void testDeleteNamespaceForciblyWithManyTopics() throws Exception { - final String ns = "prop-xyz/ns-testDeleteNamespaceForciblyWithManyTopics"; + final String ns = defaultTenant + "/ns-testDeleteNamespaceForciblyWithManyTopics"; admin.namespaces().createNamespace(ns, 2); for (int i = 0; i < 100; i++) { admin.topics().createPartitionedTopic(String.format("persistent://%s", ns + "/topic" + i), 3); } admin.namespaces().deleteNamespace(ns, true); - Assert.assertFalse(admin.namespaces().getNamespaces("prop-xyz").contains(ns)); + Assert.assertFalse(admin.namespaces().getNamespaces(defaultTenant).contains(ns)); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java index e170425ffe443..062047c7133f7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java @@ -290,6 +290,10 @@ protected void defaultOverrideServiceConfiguration(ServiceConfiguration svcConfi if (svcConfig.getManagedLedgerCacheSizeMB() == unconfiguredDefaults.getManagedLedgerCacheSizeMB()) { svcConfig.setManagedLedgerCacheSizeMB(8); } + + if (svcConfig.getTopicLoadTimeoutSeconds() == unconfiguredDefaults.getTopicLoadTimeoutSeconds()) { + svcConfig.setTopicLoadTimeoutSeconds(10); + } } /** From 6bea518f1c4795ef3577de6b12f89ecddaf295a2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 7 Jun 2023 22:51:39 +0300 Subject: [PATCH 148/494] [improve][build] Upgrade Testcontainers to 1.18.3 & docker-java to 3.3.0 (#20531) (cherry picked from commit fe556ab8ac268543004b854dbd7a59e20756dff7) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index cd82fd9ce3958..1feadb9d5b325 100644 --- a/pom.xml +++ b/pom.xml @@ -250,11 +250,11 @@ flexible messaging model and an intuitive client API. 2.0.6 - 1.17.6 + 1.18.3 2.2 - 3.2.13 + 3.3.0 1.1.1 7.7.1 3.12.4 From fb7fb8f13f3f00189b758f0a60ea388ad693a389 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 7 Jun 2023 22:52:13 +0300 Subject: [PATCH 149/494] [fix][broker] Disable EntryFilters for system topics (#20514) (cherry picked from commit ac46e2e4fc48dff74233623afa3635ef5285e34d) --- .../apache/pulsar/broker/service/AbstractTopic.java | 4 ++++ .../pulsar/broker/service/EntryFilterSupport.java | 3 ++- .../broker/service/persistent/SystemTopic.java | 13 +++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 4614b846c8eee..1371019be41dc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -1163,6 +1163,10 @@ public void updateResourceGroupLimiter(Optional optPolicies) { } public void updateEntryFilters() { + if (isSystemTopic()) { + entryFilters = Pair.of(null, Collections.emptyList()); + return; + } final EntryFilters entryFiltersPolicy = getEntryFiltersPolicy(); if (entryFiltersPolicy == null || StringUtils.isBlank(entryFiltersPolicy.getEntryFilterNames())) { entryFilters = Pair.of(null, Collections.emptyList()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java index 4a9b33a9afdb1..03d6f0750e02e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/EntryFilterSupport.java @@ -35,7 +35,8 @@ public class EntryFilterSupport { public EntryFilterSupport(Subscription subscription) { this.subscription = subscription; - if (subscription != null && subscription.getTopic() != null) { + if (subscription != null && subscription.getTopic() != null + && !subscription.getTopic().isSystemTopic()) { final BrokerService brokerService = subscription.getTopic().getBrokerService(); final boolean allowOverrideEntryFilters = brokerService .pulsar().getConfiguration().isAllowOverrideEntryFilters(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java index 395a8c9075eb3..720ae3c51891e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SystemTopic.java @@ -18,13 +18,16 @@ */ package org.apache.pulsar.broker.service.persistent; +import java.util.List; import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.plugin.EntryFilter; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.EntryFilters; public class SystemTopic extends PersistentTopic { @@ -82,4 +85,14 @@ public boolean isEncryptionRequired() { // System topics are only written by the broker that can't know the encryption context. return false; } + + @Override + public EntryFilters getEntryFiltersPolicy() { + return null; + } + + @Override + public List getEntryFilters() { + return null; + } } From e7f9a5f6a09d6b678812a4d0bb2fb2e1f3741ef1 Mon Sep 17 00:00:00 2001 From: maanders-tibco <84395784+maanders-tibco@users.noreply.github.com> Date: Thu, 8 Jun 2023 21:29:00 -0500 Subject: [PATCH 150/494] [fix][broker] REST Client Producer fails with TLS only (#20535) Co-authored-by: Matt Anderson <> Fixes #20536 ### Motivation When disabling HTTP ports in the Pulsar broker, the [REST Client Producer](https://pulsar.apache.org/docs/3.0.x/client-libraries-rest/) fails to produce messages. For reproduction steps, please reference issue details. ### Modifications Change the following lines: ```java LookupResult result = optionalResult.get(); if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress())) { // Current broker owns the topic, add to owning topic. ``` To: ```java LookupResult result = optionalResult.get(); if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress()) || result.getLookupData().getHttpUrlTls().equals(pulsar().getWebServiceAddressTls())) { // Current broker owns the topic, add to owning topic. ``` ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. (outside of the reproduction tests described in the #20536) *If the box was checked, please highlight the changes* - [ ] Dependencies (add or upgrade a dependency) - [ ] The public API - [ ] The schema - [ ] The default values of configurations - [ ] The threading model - [ ] The binary protocol - [x] The REST endpoints - [ ] The admin CLI options - [ ] The metrics - [ ] Anything that affects deployment ### Documentation - [ ] `doc` - [ ] `doc-required` - [x] `doc-not-needed` - [ ] `doc-complete` ### Matching PR in forked repository PR in forked repository: https://github.com/maanders-tibco/pulsar (cherry picked from commit 005cce11d0d67cdf7b08305bed55cd2f9ba7f4f0) --- .../main/java/org/apache/pulsar/broker/rest/TopicsBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 7d3aa37fa4ec5..33676a2c1657f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -432,7 +432,8 @@ private CompletableFuture lookUpBrokerForTopic(TopicName partitionedTopicN } LookupResult result = optionalResult.get(); - if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress())) { + if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress()) + || result.getLookupData().getHttpUrlTls().equals(pulsar().getWebServiceAddressTls())) { // Current broker owns the topic, add to owning topic. if (log.isDebugEnabled()) { log.debug("Complete topic look up for rest produce message request for topic {}, " From f4e2b2b4c7c4f42469f4864c521397d275d21eae Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 9 Jun 2023 11:42:29 +0300 Subject: [PATCH 151/494] [fix][build] Configure git-commit-id-plugin to skip git describe (#20550) (cherry picked from commit 05f7e62cc0ebe50f212559f0aac19d946eb1478c) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1feadb9d5b325..0409c4524bebe 100644 --- a/pom.xml +++ b/pom.xml @@ -1587,10 +1587,10 @@ flexible messaging model and an intuitive client API. true git false + false false - false - false + true From d681851308ca5d85990fb3856bf81bca379260da Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 9 Jun 2023 16:39:23 -0500 Subject: [PATCH 152/494] [cleanup][broker] Validate authz earlier in delete subscription logic (#20549) ### Motivation Move the authorization check a few steps earlier in the delete subscription admin endpoint. ### Modifications * Move the authz check earlier ### Verifying this change We do not have any tests for these endpoints. We should add them. This change is trivial enough that I think it is fine to defer on testing the authz change. ### Documentation - [x] `doc-not-needed` (cherry picked from commit c73967c811f60d4cb508e8489e6faf39dd0174b4) --- .../pulsar/broker/admin/impl/PersistentTopicsBase.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 5cd6ff8cbd955..81f5e3c1f323b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -1631,7 +1631,9 @@ protected CompletableFuture internalDeleteSubscriptionAsync(String subName future = CompletableFuture.completedFuture(null); } - return future.thenCompose(__ -> { + return future + .thenCompose((__) -> validateTopicOperationAsync(topicName, TopicOperation.UNSUBSCRIBE, subName)) + .thenCompose(__ -> { if (topicName.isPartitioned()) { return internalDeleteSubscriptionForNonPartitionedTopicAsync(subName, authoritative, force); } else { @@ -1674,11 +1676,11 @@ protected CompletableFuture internalDeleteSubscriptionAsync(String subName }); } + // Note: this method expects the caller to check authorization private CompletableFuture internalDeleteSubscriptionForNonPartitionedTopicAsync(String subName, boolean authoritative, boolean force) { return validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose((__) -> validateTopicOperationAsync(topicName, TopicOperation.UNSUBSCRIBE, subName)) .thenCompose(__ -> getTopicReferenceAsync(topicName)) .thenCompose((topic) -> { Subscription sub = topic.getSubscription(subName); From 7ec3a5789dee2eddbbb2621ea939d0d24de8961e Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Tue, 13 Jun 2023 15:02:59 +0800 Subject: [PATCH 153/494] [fix][authentication] Improve AuthenticationFilter response (#19464) Signed-off-by: Zixuan Liu Co-authored-by: tison (cherry picked from commit 491624395033c643f501e05276e23383f2fee61a) --- .../broker/web/AuthenticationFilter.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/AuthenticationFilter.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/AuthenticationFilter.java index 6f13185ca7540..0670412e105a0 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/AuthenticationFilter.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/web/AuthenticationFilter.java @@ -52,19 +52,29 @@ public AuthenticationFilter(AuthenticationService authenticationService) { @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + boolean allowed = false; + Exception authenticationException = null; try { - boolean doFilter = authenticationService + allowed = authenticationService .authenticateHttpRequest((HttpServletRequest) request, (HttpServletResponse) response); - if (doFilter) { - chain.doFilter(request, response); - } } catch (Exception e) { + authenticationException = e; + } + + if (allowed) { + chain.doFilter(request, response); + return; + } + + if (authenticationException != null) { HttpServletResponse httpResponse = (HttpServletResponse) response; httpResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Authentication required"); - if (e instanceof AuthenticationException) { - LOG.warn("[{}] Failed to authenticate HTTP request: {}", request.getRemoteAddr(), e.getMessage()); + if (authenticationException instanceof AuthenticationException) { + LOG.warn("[{}] Failed to authenticate HTTP request: {}", request.getRemoteAddr(), + authenticationException.getMessage()); } else { - LOG.error("[{}] Error performing authentication for HTTP", request.getRemoteAddr(), e); + LOG.error("[{}] Error performing authentication for HTTP", request.getRemoteAddr(), + authenticationException); } } } From 7e09cc3694d8645ab020f0bea593d7be1dddff13 Mon Sep 17 00:00:00 2001 From: csthomas1 <70978249+csthomas1@users.noreply.github.com> Date: Tue, 13 Jun 2023 00:35:49 -0400 Subject: [PATCH 154/494] [fix][fn] Make KubernetesRuntime translate characters in function tenant, namespace, and name during function removal to avoid label errors (#19584) Co-authored-by: tison --- .../runtime/kubernetes/KubernetesRuntime.java | 7 +- .../kubernetes/KubernetesRuntimeTest.java | 86 ++++++++++++++++--- 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java index 36c5a17ee6595..77fa1cb14e65c 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntime.java @@ -684,10 +684,11 @@ public void deleteStatefulSet() throws InterruptedException { .numRetries(KubernetesRuntimeFactory.numRetries * 2) .sleepBetweenInvocationsMs(KubernetesRuntimeFactory.sleepBetweenRetriesMs * 2) .supplier(() -> { + Map validLabels = getLabels(instanceConfig.getFunctionDetails()); String labels = String.format("tenant=%s,namespace=%s,name=%s", - instanceConfig.getFunctionDetails().getTenant(), - instanceConfig.getFunctionDetails().getNamespace(), - instanceConfig.getFunctionDetails().getName()); + validLabels.get("tenant"), + validLabels.get("namespace"), + validLabels.get("name")); V1PodList response; try { diff --git a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java index 31f82adfe8c26..7fa279bc1d253 100644 --- a/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java +++ b/pulsar-functions/runtime/src/test/java/org/apache/pulsar/functions/runtime/kubernetes/KubernetesRuntimeTest.java @@ -24,16 +24,32 @@ import com.google.gson.JsonObject; import com.google.protobuf.util.JsonFormat; import io.kubernetes.client.custom.Quantity; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.AppsV1Api; +import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Container; +import io.kubernetes.client.openapi.models.V1PodList; import io.kubernetes.client.openapi.models.V1PodSpec; import io.kubernetes.client.openapi.models.V1PodTemplateSpec; import io.kubernetes.client.openapi.models.V1ResourceRequirements; import io.kubernetes.client.openapi.models.V1Service; import io.kubernetes.client.openapi.models.V1StatefulSet; import io.kubernetes.client.openapi.models.V1Toleration; +import java.lang.reflect.Type; +import java.math.BigDecimal; +import java.net.HttpURLConnection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import okhttp3.Call; +import okhttp3.Response; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.JavaVersion; +import org.apache.commons.lang3.SystemUtils; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.functions.instance.InstanceConfig; @@ -49,27 +65,26 @@ import org.apache.pulsar.functions.worker.ConnectorsManager; import org.apache.pulsar.functions.worker.FunctionsManager; import org.apache.pulsar.functions.worker.WorkerConfig; +import org.mockito.ArgumentMatcher; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; - -import java.lang.reflect.Type; -import java.math.BigDecimal; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Consumer; -import java.util.stream.Collectors; - import static org.apache.pulsar.functions.runtime.RuntimeUtils.FUNCTIONS_INSTANCE_CLASSPATH; import static org.apache.pulsar.functions.utils.FunctionCommon.roundDecimal; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; @@ -1290,4 +1305,51 @@ private void assertMetricsPortConfigured(Map functionRuntimeFact .contains("--metrics_port 0")); } } + + @Test + public void testDeleteStatefulSetWithTranslatedKubernetesLabelChars() throws Exception { + InstanceConfig config = createJavaInstanceConfig(FunctionDetails.Runtime.JAVA, false); + config.setFunctionDetails(createFunctionDetails(FunctionDetails.Runtime.JAVA, false, + (fb) -> fb.setTenant("c:tenant").setNamespace("c:ns").setName("c:fn"))); + + CoreV1Api coreApi = mock(CoreV1Api.class); + AppsV1Api appsApi = mock(AppsV1Api.class); + + Call successfulCall = mock(Call.class); + Response okResponse = mock(Response.class); + when(okResponse.code()).thenReturn(HttpURLConnection.HTTP_OK); + when(okResponse.isSuccessful()).thenReturn(true); + when(okResponse.message()).thenReturn(""); + when(successfulCall.execute()).thenReturn(okResponse); + + final String expectedFunctionNamePrefix = String.format("pf-%s-%s-%s", "c-tenant", "c-ns", "c-fn"); + + factory = createKubernetesRuntimeFactory(null, 10, 1.0, 1.0); + factory.setCoreClient(coreApi); + factory.setAppsClient(appsApi); + + ArgumentMatcher hasTranslatedFunctionName = (String t) -> t.startsWith(expectedFunctionNamePrefix); + + when(appsApi.deleteNamespacedStatefulSetCall( + argThat(hasTranslatedFunctionName), + anyString(), isNull(), isNull(), anyInt(), isNull(), anyString(), any(), isNull())).thenReturn(successfulCall); + + ApiException notFoundException = mock(ApiException.class); + when(notFoundException.getCode()).thenReturn(HttpURLConnection.HTTP_NOT_FOUND); + when(appsApi.readNamespacedStatefulSet( + argThat(hasTranslatedFunctionName), anyString(), isNull())).thenThrow(notFoundException); + + V1PodList podList = mock(V1PodList.class); + when(podList.getItems()).thenReturn(Collections.emptyList()); + + String expectedLabels = String.format("tenant=%s,namespace=%s,name=%s", "c-tenant", "c-ns", "c-fn"); + + when(coreApi.listNamespacedPod(anyString(), isNull(), isNull(), isNull(), isNull(), + eq(expectedLabels), isNull(), isNull(), isNull(), isNull(), isNull())).thenReturn(podList); + KubernetesRuntime kr = factory.createContainer(config, "/test/code", "code.yml", "/test/transforms", "transform.yml", Long.MIN_VALUE); + kr.deleteStatefulSet(); + + verify(coreApi).listNamespacedPod(anyString(), isNull(), isNull(), isNull(), isNull(), + eq(expectedLabels), isNull(), isNull(), isNull(), isNull(), isNull()); + } } From b19ab1fad66b01b9d6fdee720d504a409381c448 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Tue, 6 Jun 2023 11:08:36 +0800 Subject: [PATCH 155/494] [fix][misc] Use ubuntu 22.04 for Pulsar images (#20475) ### Motivation Upgrade Ubuntu to 22.04 to fix some CVEs - CVE-2017-14063: https://nvd.nist.gov/vuln/detail/cve-2017-14063 - CVE-2021-27291: https://nvd.nist.gov/vuln/detail/CVE-2021-27291 - CVE-2021-20270: https://nvd.nist.gov/vuln/detail/CVE-2021-20270 --- build/docker/Dockerfile | 2 +- docker/pulsar/Dockerfile | 2 +- pom.xml | 2 +- tests/docker-images/java-test-image/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 7660325567748..3d9fc9c832bda 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM ubuntu:20.04 +FROM ubuntu:22.04 # prepare the directory for pulsar related files RUN mkdir /pulsar diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 01e53e0152ac6..a5b294063d376 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -49,7 +49,7 @@ RUN chmod g+w /pulsar/trino ### Create 2nd stage from Ubuntu image ### and add OpenJDK and Python dependencies (for Pulsar functions) -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive ARG UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt diff --git a/pom.xml b/pom.xml index 0409c4524bebe..c1e6454eb6682 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ flexible messaging model and an intuitive client API. ${maven.compiler.target} 8 - 2.10.1 + 3.2.0 **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index 9e852050edf2e..5821e6eeaeae7 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -17,7 +17,7 @@ # under the License. # -FROM ubuntu:20.04 +FROM ubuntu:22.04 RUN groupadd -g 10001 pulsar RUN adduser -u 10000 --gid 10001 --disabled-login --disabled-password --gecos '' pulsar From 692c182f6aa004e979586fbda5c2b53f8be0236b Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Tue, 13 Jun 2023 19:01:02 -0700 Subject: [PATCH 156/494] [fix][broker] new load balancer system topic should not be auto-created now (#20566) --- .../apache/pulsar/broker/service/BrokerService.java | 12 +++++++++++- .../pulsar/broker/service/BrokerServiceTest.java | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 4a01fec9abaa7..3c25935e67090 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -100,6 +100,7 @@ import org.apache.pulsar.broker.intercept.BrokerInterceptor; import org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl; import org.apache.pulsar.broker.loadbalance.LoadManager; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.resources.DynamicConfigurationResources; import org.apache.pulsar.broker.resources.LocalPoliciesResources; @@ -3291,10 +3292,19 @@ private CompletableFuture isAllowAutoTopicCreationAsync(final TopicName topicName.getNamespaceObject()); return CompletableFuture.completedFuture(false); } - //System topic can always be created automatically + + // ServiceUnitStateChannelImpl.TOPIC expects to be a non-partitioned-topic now. + // We don't allow the auto-creation here. + // ServiceUnitStateChannelImpl.start() is responsible to create the topic. + if (ServiceUnitStateChannelImpl.TOPIC.equals(topicName.toString())) { + return CompletableFuture.completedFuture(false); + } + + //Other system topics can be created automatically if (pulsar.getConfiguration().isSystemTopicEnabled() && isSystemTopic(topicName)) { return CompletableFuture.completedFuture(true); } + final boolean allowed; AutoTopicCreationOverride autoTopicCreationOverride = getAutoTopicCreationOverride(topicName, policies); if (autoTopicCreationOverride != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 24e38438c5329..4a0800dbafe83 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -72,6 +72,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -1520,4 +1521,13 @@ public void testDynamicConfigurationsForceDeleteTenantAllowed() throws Exception assertTrue(conf.isForceDeleteTenantAllowed()); }); } + + @Test + public void testIsSystemTopicAllowAutoTopicCreationAsync() throws Exception { + BrokerService brokerService = pulsar.getBrokerService(); + assertFalse(brokerService.isAllowAutoTopicCreationAsync( + ServiceUnitStateChannelImpl.TOPIC).get()); + assertTrue(brokerService.isAllowAutoTopicCreationAsync( + "persistent://pulsar/system/my-system-topic").get()); + } } From 52e57a6e49fb5a1055476913ad6e6bc3f430c14b Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 8 Jun 2023 13:58:08 +0800 Subject: [PATCH 157/494] [fix][broker] Fix redirect loop when using ExtensibleLoadManager and list in bundle admin API (#20528) PIP: https://github.com/apache/pulsar/issues/16691 When using `ExtensibleLoadManager` and list in bundle admin API, it will redirect forever because `isServiceUnitOwned` method is checking the `ownershipCache` as the ownership storage, however, when using `ExtensibleLoadManager`, it stored the ownership to table view. * Call `isServiceUnitOwnedAsync ` when using `isServiceUnitOwned `. * Add unit test to cover this case. --- .../broker/namespace/NamespaceService.java | 18 +------- .../ExtensibleLoadManagerImplTest.java | 41 ++++++++++++++++++- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index e2d4ef5153769..c5d516cfc944c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1102,19 +1102,7 @@ public Set getOwnedServiceUnits() { } public boolean isServiceUnitOwned(ServiceUnitId suName) throws Exception { - if (suName instanceof TopicName) { - return isTopicOwnedAsync((TopicName) suName).get(); - } - - if (suName instanceof NamespaceName) { - return isNamespaceOwned((NamespaceName) suName); - } - - if (suName instanceof NamespaceBundle) { - return ownershipCache.isNamespaceBundleOwned((NamespaceBundle) suName); - } - - throw new IllegalArgumentException("Invalid class of NamespaceBundle: " + suName.getClass().getName()); + return isServiceUnitOwnedAsync(suName).get(config.getMetadataStoreOperationTimeoutSeconds(), TimeUnit.SECONDS); } public CompletableFuture isServiceUnitOwnedAsync(ServiceUnitId suName) { @@ -1168,10 +1156,6 @@ public CompletableFuture isServiceUnitActiveAsync(TopicName topicName) }); } - private boolean isNamespaceOwned(NamespaceName fqnn) throws Exception { - return ownershipCache.getOwnedBundle(getFullBundle(fqnn)) != null; - } - private CompletableFuture isNamespaceOwnedAsync(NamespaceName fqnn) { // TODO: Add unit tests cover it. if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 498f48d16d2cd..d6e208d3202c0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -48,10 +48,11 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import com.google.common.collect.Sets; import java.util.LinkedHashMap; import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -63,7 +64,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -927,6 +927,43 @@ public void testDisableBroker() throws Exception { } } + @Test(timeOut = 30 * 1000) + public void testListTopic() throws Exception { + final String namespace = "public/testListTopic"; + admin.namespaces().createNamespace(namespace, 3); + + final String persistentTopicName = TopicName.get( + "persistent", NamespaceName.get(namespace), + "get_topics_mode_" + UUID.randomUUID()).toString(); + + final String nonPersistentTopicName = TopicName.get( + "non-persistent", NamespaceName.get(namespace), + "get_topics_mode_" + UUID.randomUUID()).toString(); + admin.topics().createPartitionedTopic(persistentTopicName, 3); + admin.topics().createPartitionedTopic(nonPersistentTopicName, 3); + pulsarClient.newProducer().topic(persistentTopicName).create().close(); + pulsarClient.newProducer().topic(nonPersistentTopicName).create().close(); + + BundlesData bundlesData = admin.namespaces().getBundles(namespace); + List boundaries = bundlesData.getBoundaries(); + int topicNum = 0; + for (int i = 0; i < boundaries.size() - 1; i++) { + String bundle = String.format("%s_%s", boundaries.get(i), boundaries.get(i + 1)); + List topic = admin.topics().getListInBundle(namespace, bundle); + if (topic == null) { + continue; + } + topicNum += topic.size(); + for (String s : topic) { + assertFalse(TopicName.get(s).isPersistent()); + } + } + assertEquals(topicNum, 3); + + List list = admin.topics().getList(namespace); + assertEquals(list.size(), 6); + } + private static abstract class MockBrokerFilter implements BrokerFilter { @Override From 99b7b1c4e3c33e942611e5b8a5efb56510ff21d1 Mon Sep 17 00:00:00 2001 From: hleecs Date: Thu, 1 Jun 2023 12:36:46 +0800 Subject: [PATCH 158/494] [feat][broker]PIP-255 Part-1: Add listener interface for namespace service (#20406) --- .../NamespaceBundleSplitListener.java | 29 ++++++++++++++++ .../broker/namespace/NamespaceService.java | 27 +++++++++++++++ .../namespace/NamespaceCreateBundlesTest.java | 33 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java new file mode 100644 index 0000000000000..a3312f5689e38 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.namespace; + +import java.util.function.Predicate; +import org.apache.pulsar.common.naming.NamespaceBundle; + +/** + * Listener for NamespaceBundle split. + */ +public interface NamespaceBundleSplitListener extends Predicate { + void onSplit(NamespaceBundle bundle); +} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index c5d516cfc944c..3997ab521b889 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -139,6 +139,10 @@ public class NamespaceService implements AutoCloseable { private final ConcurrentOpenHashMap namespaceClients; private final List bundleOwnershipListeners; + + private final List bundleSplitListeners; + + private final RedirectManager redirectManager; @@ -167,6 +171,7 @@ public NamespaceService(PulsarService pulsar) { this.namespaceClients = ConcurrentOpenHashMap.newBuilder().build(); this.bundleOwnershipListeners = new CopyOnWriteArrayList<>(); + this.bundleSplitListeners = new CopyOnWriteArrayList<>(); this.localBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalBrokerData.class); this.redirectManager = new RedirectManager(pulsar); } @@ -975,6 +980,7 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, // affect the split operation which is already safely completed r.forEach(this::unloadNamespaceBundle); } + onNamespaceBundleSplit(bundle); }) .exceptionally(e -> { String msg1 = format( @@ -1214,6 +1220,18 @@ protected void onNamespaceBundleUnload(NamespaceBundle bundle) { } } + protected void onNamespaceBundleSplit(NamespaceBundle bundle) { + for (NamespaceBundleSplitListener bundleSplitListener : bundleSplitListeners) { + try { + if (bundleSplitListener.test(bundle)) { + bundleSplitListener.onSplit(bundle); + } + } catch (Throwable t) { + LOG.error("Call bundle {} split listener {} error", bundle, bundleSplitListener, t); + } + } + } + public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener... listeners) { Objects.requireNonNull(listeners); for (NamespaceBundleOwnershipListener listener : listeners) { @@ -1224,6 +1242,15 @@ public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener getOwnedServiceUnits().forEach(bundle -> notifyNamespaceBundleOwnershipListener(bundle, listeners)); } + public void addNamespaceBundleSplitListener(NamespaceBundleSplitListener... listeners) { + Objects.requireNonNull(listeners); + for (NamespaceBundleSplitListener listener : listeners) { + if (listener != null) { + bundleSplitListeners.add(listener); + } + } + } + private void notifyNamespaceBundleOwnershipListener(NamespaceBundle bundle, NamespaceBundleOwnershipListener... listeners) { if (listeners != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java index 43d37466918ce..73cfaf1b0d96b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java @@ -20,15 +20,21 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import lombok.Cleanup; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.Policies; +import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -81,4 +87,31 @@ public void testSplitBundleUpdatesLocalPoliciesWithoutOverwriting() throws Excep assertNotNull(admin.namespaces().getBookieAffinityGroup(namespaceName)); producer.close(); } + + @Test + public void testBundleSplitListener() throws Exception { + String namespaceName = "prop/" + UUID.randomUUID().toString(); + String topicName = "persistent://" + namespaceName + "/my-topic5"; + admin.namespaces().createNamespace(namespaceName); + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topicName).sendTimeout(1, + TimeUnit.SECONDS).create(); + producer.send(new byte[1]); + String bundleRange = admin.lookups().getBundleRange(topicName); + AtomicBoolean isTriggered = new AtomicBoolean(false); + pulsar.getNamespaceService().addNamespaceBundleSplitListener(new NamespaceBundleSplitListener() { + @Override + public void onSplit(NamespaceBundle bundle) { + assertEquals(bundleRange, bundle.getBundleRange()); + isTriggered.set(true); + } + + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return true; + } + }); + admin.namespaces().splitNamespaceBundle(namespaceName, bundleRange, false, null); + Awaitility.await().untilAsserted(() -> assertTrue(isTriggered.get())); + } } From 32ba7470f36c52abaa59a31d7e7463d238346df6 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Fri, 9 Jun 2023 10:40:27 +0800 Subject: [PATCH 159/494] [improve][broker] Emit the namespace bundle listener event on extensible load manager (#20525) --- .../channel/ServiceUnitStateChannelImpl.java | 3 + .../broker/namespace/NamespaceService.java | 6 +- .../ExtensibleLoadManagerImplTest.java | 103 +++++++++++++++--- 3 files changed, 93 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 489a00851057b..48192b5dc7cd8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -703,6 +703,7 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.dstBroker())) { log(null, serviceUnit, data, null); + pulsar.getNamespaceService().onNamespaceBundleOwned(getNamespaceBundle(serviceUnit)); lastOwnEventHandledAt = System.currentTimeMillis(); } else if (data.force() && isTargetBroker(data.sourceBroker())) { closeServiceUnit(serviceUnit); @@ -841,6 +842,7 @@ private CompletableFuture closeServiceUnit(String serviceUnit) { .whenComplete((__, ex) -> { // clean up topics that failed to unload from the broker ownership cache pulsar.getBrokerService().cleanUnloadedTopicFromCache(bundle); + pulsar.getNamespaceService().onNamespaceBundleUnload(bundle); double unloadBundleTime = TimeUnit.NANOSECONDS .toMillis((System.nanoTime() - startTime)); if (ex != null) { @@ -912,6 +914,7 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)); log.info("Successfully split {} parent namespace-bundle to {} in {} ms", parentBundle, childBundles, splitBundleTime); + namespaceService.onNamespaceBundleSplit(parentBundle); completionFuture.complete(null); }) .exceptionally(ex -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 3997ab521b889..69c25d7c6d394 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1202,13 +1202,13 @@ public CompletableFuture removeOwnedServiceUnitAsync(NamespaceBundle nsBun return future.thenRun(() -> bundleFactory.invalidateBundleCache(nsBundle.getNamespaceObject())); } - protected void onNamespaceBundleOwned(NamespaceBundle bundle) { + public void onNamespaceBundleOwned(NamespaceBundle bundle) { for (NamespaceBundleOwnershipListener bundleOwnedListener : bundleOwnershipListeners) { notifyNamespaceBundleOwnershipListener(bundle, bundleOwnedListener); } } - protected void onNamespaceBundleUnload(NamespaceBundle bundle) { + public void onNamespaceBundleUnload(NamespaceBundle bundle) { for (NamespaceBundleOwnershipListener bundleOwnedListener : bundleOwnershipListeners) { try { if (bundleOwnedListener.test(bundle)) { @@ -1220,7 +1220,7 @@ protected void onNamespaceBundleUnload(NamespaceBundle bundle) { } } - protected void onNamespaceBundleSplit(NamespaceBundle bundle) { + public void onNamespaceBundleSplit(NamespaceBundle bundle) { for (NamespaceBundleSplitListener bundleSplitListener : bundleSplitListeners) { try { if (bundleSplitListener.test(bundle)) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index d6e208d3202c0..8d7ebf8bfe3cf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -87,6 +87,8 @@ import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; +import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; +import org.apache.pulsar.broker.namespace.NamespaceBundleSplitListener; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.TableViewImpl; @@ -126,6 +128,8 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { private ServiceUnitStateChannelImpl channel1; private ServiceUnitStateChannelImpl channel2; + private final String defaultTestNamespace = "public/test"; + @BeforeClass @Override public void setup() throws Exception { @@ -136,6 +140,7 @@ public void setup() throws Exception { conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); conf.setLoadBalancerSheddingEnabled(false); conf.setLoadBalancerDebugModeEnabled(true); + conf.setTopicLevelPoliciesEnabled(false); super.internalSetup(conf); pulsar1 = pulsar; ServiceConfiguration defaultConf = getDefaultConf(); @@ -144,6 +149,7 @@ public void setup() throws Exception { defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); defaultConf.setLoadBalancerSheddingEnabled(false); + defaultConf.setTopicLevelPoliciesEnabled(false); additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); pulsar2 = additionalPulsarTestContext.getPulsarService(); @@ -159,6 +165,10 @@ public void setup() throws Exception { admin.namespaces().createNamespace("public/default"); admin.namespaces().setNamespaceReplicationClusters("public/default", Sets.newHashSet(this.conf.getClusterName())); + + admin.namespaces().createNamespace(defaultTestNamespace); + admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace, + Sets.newHashSet(this.conf.getClusterName())); } @Override @@ -172,7 +182,7 @@ protected void cleanup() throws Exception { @BeforeMethod(alwaysRun = true) protected void initializeState() throws PulsarAdminException { - admin.namespaces().unload("public/default"); + admin.namespaces().unload(defaultTestNamespace); reset(primaryLoadManager, secondaryLoadManager); } @@ -196,7 +206,7 @@ public void testAssignInternalTopic() throws Exception { @Test public void testAssign() throws Exception { - TopicName topicName = TopicName.get("test-assign"); + TopicName topicName = TopicName.get(defaultTestNamespace + "/test-assign"); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); Optional brokerLookupData = primaryLoadManager.assign(Optional.empty(), bundle).get(); assertTrue(brokerLookupData.isPresent()); @@ -221,7 +231,8 @@ public void testAssign() throws Exception { @Test public void testCheckOwnershipAsync() throws Exception { - NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get("test-check-ownership")).get(); + TopicName topicName = TopicName.get(defaultTestNamespace + "/test-check-ownership"); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); // 1. The bundle is never assigned. assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); @@ -241,7 +252,7 @@ public void testCheckOwnershipAsync() throws Exception { @Test public void testFilter() throws Exception { - TopicName topicName = TopicName.get("test-filter"); + TopicName topicName = TopicName.get(defaultTestNamespace + "/test-filter"); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); doReturn(List.of(new BrokerFilter() { @@ -267,7 +278,7 @@ public Map filter(Map broker @Test public void testFilterHasException() throws Exception { - TopicName topicName = TopicName.get("test-filter-has-exception"); + TopicName topicName = TopicName.get(defaultTestNamespace + "/test-filter-has-exception"); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); doReturn(List.of(new MockBrokerFilter() { @@ -286,20 +297,58 @@ public Map filter(Map broker @Test(timeOut = 30 * 1000) public void testUnloadAdminAPI() throws Exception { - TopicName topicName = TopicName.get("test-unload"); + TopicName topicName = TopicName.get(defaultTestNamespace + "/test-unload"); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + AtomicInteger onloadCount = new AtomicInteger(0); + AtomicInteger unloadCount = new AtomicInteger(0); + + NamespaceBundleOwnershipListener listener = new NamespaceBundleOwnershipListener() { + @Override + public void onLoad(NamespaceBundle bundle) { + onloadCount.incrementAndGet(); + } + + @Override + public void unLoad(NamespaceBundle bundle) { + unloadCount.incrementAndGet(); + } + + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return namespaceBundle.equals(bundle); + } + }; + pulsar1.getNamespaceService().addNamespaceBundleOwnershipListener(listener); + pulsar2.getNamespaceService().addNamespaceBundleOwnershipListener(listener); String broker = admin.lookups().lookupTopic(topicName.toString()); log.info("Assign the bundle {} to {}", bundle, broker); checkOwnershipState(broker, bundle); + Awaitility.await().untilAsserted(() -> { + assertEquals(onloadCount.get(), 1); + assertEquals(unloadCount.get(), 0); + }); + admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange()); assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); assertFalse(secondaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + Awaitility.await().untilAsserted(() -> { + assertEquals(onloadCount.get(), 1); + assertEquals(unloadCount.get(), 1); + }); broker = admin.lookups().lookupTopic(topicName.toString()); log.info("Assign the bundle {} to {}", bundle, broker); + String finalBroker = broker; + Awaitility.await().untilAsserted(() -> { + checkOwnershipState(finalBroker, bundle); + assertEquals(onloadCount.get(), 2); + assertEquals(unloadCount.get(), 1); + }); + + String dstBrokerUrl = pulsar1.getLookupServiceAddress(); String dstBrokerServiceUrl; if (broker.equals(pulsar1.getBrokerServiceUrl())) { @@ -311,6 +360,10 @@ public void testUnloadAdminAPI() throws Exception { checkOwnershipState(broker, bundle); admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), dstBrokerUrl); + Awaitility.await().untilAsserted(() -> { + assertEquals(onloadCount.get(), 3); + assertEquals(unloadCount.get(), 2); + }); assertEquals(admin.lookups().lookupTopic(topicName.toString()), dstBrokerServiceUrl); @@ -338,7 +391,7 @@ private void checkOwnershipState(String broker, NamespaceBundle bundle) @Test(timeOut = 30 * 1000) public void testSplitBundleAdminAPI() throws Exception { - String namespace = "public/default"; + String namespace = defaultTestNamespace; String topic = "persistent://" + namespace + "/test-split"; admin.topics().createPartitionedTopic(topic, 10); BundlesData bundles = admin.namespaces().getBundles(namespace); @@ -347,6 +400,23 @@ public void testSplitBundleAdminAPI() throws Exception { String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1); + AtomicInteger splitCount = new AtomicInteger(0); + NamespaceBundleSplitListener namespaceBundleSplitListener = new NamespaceBundleSplitListener() { + @Override + public void onSplit(NamespaceBundle bundle) { + splitCount.incrementAndGet(); + } + + @Override + public boolean test(NamespaceBundle namespaceBundle) { + return namespaceBundle + .toString() + .equals(String.format(namespace + "/0x%08x_0x%08x", bundleRanges.get(0), bundleRanges.get(1))); + } + }; + pulsar1.getNamespaceService().addNamespaceBundleSplitListener(namespaceBundleSplitListener); + pulsar2.getNamespaceService().addNamespaceBundleSplitListener(namespaceBundleSplitListener); + long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2; admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, null); @@ -359,6 +429,7 @@ public void testSplitBundleAdminAPI() throws Exception { assertTrue(bundlesData.getBoundaries().contains(lowBundle)); assertTrue(bundlesData.getBoundaries().contains(midBundle)); assertTrue(bundlesData.getBoundaries().contains(highBundle)); + assertEquals(splitCount.get(), 1); // Test split bundle with invalid bundle range. try { @@ -371,7 +442,7 @@ public void testSplitBundleAdminAPI() throws Exception { @Test(timeOut = 30 * 1000) public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { - String namespace = "public/default"; + String namespace = defaultTestNamespace; String topic = "persistent://" + namespace + "/test-split-with-specific-position"; admin.topics().createPartitionedTopic(topic, 10); BundlesData bundles = admin.namespaces().getBundles(namespace); @@ -398,7 +469,9 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { } @Test(timeOut = 30 * 1000) public void testDeleteNamespaceBundle() throws Exception { - TopicName topicName = TopicName.get("test-delete-namespace-bundle"); + final String namespace = "public/testDeleteNamespaceBundle"; + admin.namespaces().createNamespace(namespace, 3); + TopicName topicName = TopicName.get(namespace + "/test-delete-namespace-bundle"); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); String broker = admin.lookups().lookupTopic(topicName.toString()); @@ -447,7 +520,7 @@ public void testCheckOwnershipPresentWithSystemNamespace() throws Exception { @Test public void testMoreThenOneFilter() throws Exception { - TopicName topicName = TopicName.get("test-filter-has-exception"); + TopicName topicName = TopicName.get(defaultTestNamespace + "/test-filter-has-exception"); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); @@ -485,7 +558,7 @@ public void testDeployAndRollbackLoadManager() throws Exception { try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { // start pulsar3 with old load manager var pulsar3 = additionalPulsarTestContext.getPulsarService(); - String topic = "persistent://public/default/test"; + String topic = "persistent://" + defaultTestNamespace + "/test"; String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl()); @@ -594,7 +667,7 @@ public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Except restartBroker(); pulsar1 = pulsar; setPrimaryLoadManager(); - admin.namespaces().setNamespaceReplicationClusters("public/default", + admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace, Sets.newHashSet(this.conf.getClusterName())); var serviceUnitStateChannelPrimaryNew = @@ -614,10 +687,7 @@ public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Except } @Test - public void testRoleChange() - throws Exception { - - + public void testRoleChange() throws Exception { var topBundlesLoadDataStorePrimary = (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true); var topBundlesLoadDataStorePrimarySpy = spy(topBundlesLoadDataStorePrimary); @@ -962,6 +1032,7 @@ public void testListTopic() throws Exception { List list = admin.topics().getList(namespace); assertEquals(list.size(), 6); + admin.namespaces().deleteNamespace(namespace, true); } private static abstract class MockBrokerFilter implements BrokerFilter { From 9dce020482a7208459985b81934c54b1120496a9 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Mon, 12 Jun 2023 20:39:05 +0800 Subject: [PATCH 160/494] [improve][broker] Handle get owned namespaces admin API in ExtensibleLoadManager (#20552) --- .../extensions/ExtensibleLoadManagerImpl.java | 25 +++++++- .../channel/ServiceUnitStateChannelImpl.java | 16 ++---- .../loadbalance/impl/LoadManagerShared.java | 6 ++ .../broker/namespace/NamespaceService.java | 57 +++++++++++++++---- .../ExtensibleLoadManagerImplTest.java | 54 +++++++++++++++++- 5 files changed, 133 insertions(+), 25 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 531ab18938a1e..4de6ccaf79c14 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -23,6 +23,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; +import static org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.getNamespaceBundle; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; @@ -37,16 +38,20 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.LoadManager; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; @@ -170,7 +175,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private final SplitCounter splitCounter = new SplitCounter(); // record unload metrics - private final AtomicReference> unloadMetrics = new AtomicReference(); + private final AtomicReference> unloadMetrics = new AtomicReference<>(); // record split metrics private final AtomicReference> splitMetrics = new AtomicReference<>(); @@ -180,6 +185,24 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { .build(); private final CountDownLatch initWaiter = new CountDownLatch(1); + /** + * Get all the bundles that are owned by this broker. + */ + public Set getOwnedServiceUnits() { + Set> entrySet = serviceUnitStateChannel.getOwnershipEntrySet(); + String brokerId = brokerRegistry.getBrokerId(); + return entrySet.stream() + .filter(entry -> { + var stateData = entry.getValue(); + return stateData.state() == ServiceUnitState.Owned + && StringUtils.isNotBlank(stateData.dstBroker()) + && stateData.dstBroker().equals(brokerId); + }).map(entry -> { + var bundle = entry.getKey(); + return getNamespaceBundle(pulsar, bundle); + }).collect(Collectors.toSet()); + } + public enum Role { Leader, Follower diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 48192b5dc7cd8..99c538e6ecfa3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -703,7 +703,8 @@ private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { stateChangeListeners.notify(serviceUnit, data, null); if (isTargetBroker(data.dstBroker())) { log(null, serviceUnit, data, null); - pulsar.getNamespaceService().onNamespaceBundleOwned(getNamespaceBundle(serviceUnit)); + pulsar.getNamespaceService() + .onNamespaceBundleOwned(LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit)); lastOwnEventHandledAt = System.currentTimeMillis(); } else if (data.force() && isTargetBroker(data.sourceBroker())) { closeServiceUnit(serviceUnit); @@ -803,12 +804,6 @@ private boolean isTargetBroker(String broker) { return broker.equals(lookupServiceAddress); } - private NamespaceBundle getNamespaceBundle(String bundle) { - final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); - final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); - return pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange); - } - private CompletableFuture deferGetOwnerRequest(String serviceUnit) { return getOwnerRequests .computeIfAbsent(serviceUnit, k -> { @@ -829,7 +824,7 @@ private CompletableFuture deferGetOwnerRequest(String serviceUnit) { private CompletableFuture closeServiceUnit(String serviceUnit) { long startTime = System.nanoTime(); MutableInt unloadedTopics = new MutableInt(); - NamespaceBundle bundle = getNamespaceBundle(serviceUnit); + NamespaceBundle bundle = LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit); return pulsar.getBrokerService().unloadServiceUnit( bundle, true, @@ -860,7 +855,7 @@ private CompletableFuture splitServiceUnit(String serviceUnit, ServiceUnit long startTime = System.nanoTime(); NamespaceService namespaceService = pulsar.getNamespaceService(); NamespaceBundleFactory bundleFactory = namespaceService.getNamespaceBundleFactory(); - NamespaceBundle bundle = getNamespaceBundle(serviceUnit); + NamespaceBundle bundle = LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit); CompletableFuture completionFuture = new CompletableFuture<>(); Map> bundleToDestBroker = data.splitServiceUnitToDestBroker(); List boundaries = null; @@ -1275,7 +1270,8 @@ private synchronized void doCleanup(String broker) { private Optional selectBroker(String serviceUnit, String inactiveBroker) { try { - return loadManager.selectAsync(getNamespaceBundle(serviceUnit), Set.of(inactiveBroker)) + return loadManager.selectAsync( + LoadManagerShared.getNamespaceBundle(pulsar, serviceUnit), Set.of(inactiveBroker)) .get(inFlightStateWaitingTimeInMillis, MILLISECONDS); } catch (Throwable e) { log.error("Failed to select a broker for serviceUnit:{}", serviceUnit); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index 6818ae03b5280..33f346adbe0c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -711,4 +711,10 @@ public static void refreshBrokerToFailureDomainMap(PulsarService pulsar, LOG.warn("Failed to get domain-list for cluster {}", e.getMessage()); } } + + public static NamespaceBundle getNamespaceBundle(PulsarService pulsar, String bundle) { + final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + return pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 69c25d7c6d394..76b4e093f7dc4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -757,21 +757,42 @@ public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle) { } public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, Optional destinationBroker) { - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { - return ExtensibleLoadManagerImpl.get(loadManager.get()) - .unloadNamespaceBundleAsync(bundle, destinationBroker); - } + // unload namespace bundle - return unloadNamespaceBundle(bundle, config.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); + return unloadNamespaceBundle(bundle, destinationBroker, + config.getNamespaceBundleUnloadingTimeoutMs(), TimeUnit.MILLISECONDS); + } + + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, + Optional destinationBroker, + long timeout, + TimeUnit timeoutUnit) { + return unloadNamespaceBundle(bundle, destinationBroker, timeout, timeoutUnit, true); + } + + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, + long timeout, + TimeUnit timeoutUnit) { + return unloadNamespaceBundle(bundle, Optional.empty(), timeout, timeoutUnit, true); } - public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, long timeout, TimeUnit timeoutUnit) { - return unloadNamespaceBundle(bundle, timeout, timeoutUnit, true); + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, + long timeout, + TimeUnit timeoutUnit, + boolean closeWithoutWaitingClientDisconnect) { + return unloadNamespaceBundle(bundle, Optional.empty(), timeout, + timeoutUnit, closeWithoutWaitingClientDisconnect); } - public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, long timeout, + public CompletableFuture unloadNamespaceBundle(NamespaceBundle bundle, + Optional destinationBroker, + long timeout, TimeUnit timeoutUnit, boolean closeWithoutWaitingClientDisconnect) { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + return ExtensibleLoadManagerImpl.get(loadManager.get()) + .unloadNamespaceBundleAsync(bundle, destinationBroker); + } // unload namespace bundle OwnedBundle ob = ownershipCache.getOwnedBundle(bundle); if (ob == null) { @@ -790,13 +811,23 @@ public CompletableFuture> getOwnedNameSpac .getIsolationDataPoliciesAsync(pulsar.getConfiguration().getClusterName()) .thenApply(nsIsolationPoliciesOpt -> nsIsolationPoliciesOpt.orElseGet(NamespaceIsolationPolicies::new)) .thenCompose(namespaceIsolationPolicies -> { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + ExtensibleLoadManagerImpl extensibleLoadManager = + ExtensibleLoadManagerImpl.get(loadManager.get()); + var statusMap = extensibleLoadManager.getOwnedServiceUnits().stream() + .collect(Collectors.toMap(NamespaceBundle::toString, + bundle -> getNamespaceOwnershipStatus(true, + namespaceIsolationPolicies.getPolicyByNamespace( + bundle.getNamespaceObject())))); + return CompletableFuture.completedFuture(statusMap); + } Collection> futures = ownershipCache.getOwnedBundlesAsync().values(); return FutureUtil.waitForAll(futures) .thenApply(__ -> futures.stream() .map(CompletableFuture::join) .collect(Collectors.toMap(bundle -> bundle.getNamespaceBundle().toString(), - bundle -> getNamespaceOwnershipStatus(bundle, + bundle -> getNamespaceOwnershipStatus(bundle.isActive(), namespaceIsolationPolicies.getPolicyByNamespace( bundle.getNamespaceBundle().getNamespaceObject())) )) @@ -804,10 +835,10 @@ public CompletableFuture> getOwnedNameSpac }); } - private NamespaceOwnershipStatus getNamespaceOwnershipStatus(OwnedBundle nsObj, + private NamespaceOwnershipStatus getNamespaceOwnershipStatus(boolean isActive, NamespaceIsolationPolicy nsIsolationPolicy) { NamespaceOwnershipStatus nsOwnedStatus = new NamespaceOwnershipStatus(BrokerAssignment.shared, false, - nsObj.isActive()); + isActive); if (nsIsolationPolicy == null) { // no matching policy found, this namespace must be an uncontrolled one and using shared broker return nsOwnedStatus; @@ -1103,6 +1134,10 @@ public OwnershipCache getOwnershipCache() { } public Set getOwnedServiceUnits() { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); + return extensibleLoadManager.getOwnedServiceUnits(); + } return ownershipCache.getOwnedBundles().values().stream().map(OwnedBundle::getNamespaceBundle) .collect(Collectors.toSet()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 8d7ebf8bfe3cf..22321cc8d35e0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -96,8 +96,10 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.BrokerAssignment; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.NamespaceOwnershipStatus; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.stats.Metrics; @@ -568,7 +570,7 @@ public void testDeployAndRollbackLoadManager() throws Exception { assertEquals(lookupResult1, lookupResult2); assertEquals(lookupResult1, lookupResult3); - NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get("test")).get(); + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).get(); LookupOptions options = LookupOptions.builder() .authoritative(false) .requestHttps(false) @@ -964,10 +966,10 @@ public void testDisableBroker() throws Exception { var pulsar3 = additionalPulsarTestContext.getPulsarService(); ExtensibleLoadManagerImpl ternaryLoadManager = spy((ExtensibleLoadManagerImpl) FieldUtils.readField(pulsar3.getLoadManager().get(), "loadManager", true)); - String topic = "persistent://public/default/test"; + String topic = "persistent://" + defaultTestNamespace +"/test"; String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); - TopicName topicName = TopicName.get("test"); + TopicName topicName = TopicName.get(topic); NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); if (!pulsar3.getBrokerServiceUrl().equals(lookupResult1)) { admin.namespaces().unloadNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), @@ -1035,6 +1037,52 @@ public void testListTopic() throws Exception { admin.namespaces().deleteNamespace(namespace, true); } + @Test(timeOut = 30 * 1000) + public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws PulsarAdminException { + Set ownedServiceUnitsByPulsar1 = primaryLoadManager.getOwnedServiceUnits(); + log.info("Owned service units: {}", ownedServiceUnitsByPulsar1); + assertTrue(ownedServiceUnitsByPulsar1.isEmpty()); + Set ownedServiceUnitsByPulsar2 = secondaryLoadManager.getOwnedServiceUnits(); + log.info("Owned service units: {}", ownedServiceUnitsByPulsar2); + assertTrue(ownedServiceUnitsByPulsar2.isEmpty()); + Map ownedNamespacesByPulsar1 = + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar1.getLookupServiceAddress()); + Map ownedNamespacesByPulsar2 = + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar2.getLookupServiceAddress()); + assertTrue(ownedNamespacesByPulsar1.isEmpty()); + assertTrue(ownedNamespacesByPulsar2.isEmpty()); + + String topic = "persistent://" + defaultTestNamespace + "/test-get-owned-service-units"; + admin.topics().createPartitionedTopic(topic, 1); + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).join(); + CompletableFuture> owner = primaryLoadManager.assign(Optional.empty(), bundle); + assertFalse(owner.join().isEmpty()); + + BrokerLookupData brokerLookupData = owner.join().get(); + if (brokerLookupData.getWebServiceUrl().equals(pulsar1.getWebServiceAddress())) { + assertOwnedServiceUnits(pulsar1, primaryLoadManager, bundle); + } else { + assertOwnedServiceUnits(pulsar2, secondaryLoadManager, bundle); + } + } + + private void assertOwnedServiceUnits( + PulsarService pulsar, + ExtensibleLoadManagerImpl extensibleLoadManager, + NamespaceBundle bundle) throws PulsarAdminException { + Awaitility.await().untilAsserted(() -> { + Set ownedBundles = extensibleLoadManager.getOwnedServiceUnits(); + assertTrue(ownedBundles.contains(bundle)); + }); + Map ownedNamespaces = + admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar.getLookupServiceAddress()); + assertTrue(ownedNamespaces.containsKey(bundle.toString())); + NamespaceOwnershipStatus status = ownedNamespaces.get(bundle.toString()); + assertTrue(status.is_active); + assertFalse(status.is_controlled); + assertEquals(status.broker_assignment, BrokerAssignment.shared); + } + private static abstract class MockBrokerFilter implements BrokerFilter { @Override From e5e853b0e9fc137a704d0b8919c2f4c9e2e1e34b Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 14 Jun 2023 13:24:50 +0800 Subject: [PATCH 161/494] [fix][broker] Handle heartbeat namespace in ExtensibleLoadManager (#20551) --- .../extensions/ExtensibleLoadManagerImpl.java | 117 ++++++++++++------ .../channel/ServiceUnitStateChannelImpl.java | 18 +++ .../broker/namespace/NamespaceService.java | 9 +- .../ExtensibleLoadManagerImplTest.java | 59 ++++++++- 4 files changed, 161 insertions(+), 42 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 4de6ccaf79c14..a9a6e6138bda9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -38,6 +38,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -84,6 +85,7 @@ import org.apache.pulsar.broker.loadbalance.extensions.strategy.LeastResourceUsageWithWeight; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; @@ -365,56 +367,99 @@ public CompletableFuture> assign(Optional> future = lookupRequests.computeIfAbsent(bundle, k -> { + return dedupeLookupRequest(bundle, k -> { final CompletableFuture> owner; // Assign the bundle to channel owner if is internal topic, to avoid circular references. if (topic.isPresent() && isInternalTopic(topic.get().toString())) { owner = serviceUnitStateChannel.getChannelOwnerAsync(); } else { - owner = serviceUnitStateChannel.getOwnerAsync(bundle).thenCompose(broker -> { - // If the bundle not assign yet, select and publish assign event to channel. - if (broker.isEmpty()) { - return this.selectAsync(serviceUnit).thenCompose(brokerOpt -> { - if (brokerOpt.isPresent()) { - assignCounter.incrementSuccess(); - log.info("Selected new owner broker: {} for bundle: {}.", brokerOpt.get(), bundle); - return serviceUnitStateChannel.publishAssignEventAsync(bundle, brokerOpt.get()) - .thenApply(Optional::of); - } else { - throw new IllegalStateException( - "Failed to select the new owner broker for bundle: " + bundle); - } - }); + owner = getOwnerAsync(serviceUnit, bundle, false).thenApply(Optional::ofNullable); + } + return getBrokerLookupData(owner, bundle); + }); + } + + private CompletableFuture getOwnerAsync( + ServiceUnitId serviceUnit, String bundle, boolean ownByLocalBrokerIfAbsent) { + return serviceUnitStateChannel.getOwnerAsync(bundle).thenCompose(broker -> { + // If the bundle not assign yet, select and publish assign event to channel. + if (broker.isEmpty()) { + CompletableFuture> selectedBroker; + if (ownByLocalBrokerIfAbsent) { + String brokerId = this.brokerRegistry.getBrokerId(); + selectedBroker = CompletableFuture.completedFuture(Optional.of(brokerId)); + } else { + selectedBroker = this.selectAsync(serviceUnit); + } + return selectedBroker.thenCompose(brokerOpt -> { + if (brokerOpt.isPresent()) { + assignCounter.incrementSuccess(); + log.info("Selected new owner broker: {} for bundle: {}.", brokerOpt.get(), bundle); + return serviceUnitStateChannel.publishAssignEventAsync(bundle, brokerOpt.get()); } - assignCounter.incrementSkip(); - // Already assigned, return it. - return CompletableFuture.completedFuture(broker); + throw new IllegalStateException( + "Failed to select the new owner broker for bundle: " + bundle); }); } + assignCounter.incrementSkip(); + // Already assigned, return it. + return CompletableFuture.completedFuture(broker.get()); + }); + } - return owner.thenCompose(broker -> { - if (broker.isEmpty()) { - String errorMsg = String.format( - "Failed to get or assign the owner for bundle:%s", bundle); - log.error(errorMsg); - throw new IllegalStateException(errorMsg); - } - return CompletableFuture.completedFuture(broker.get()); - }).thenCompose(broker -> this.getBrokerRegistry().lookupAsync(broker).thenCompose(brokerLookupData -> { - if (brokerLookupData.isEmpty()) { - String errorMsg = String.format( - "Failed to look up a broker registry:%s for bundle:%s", broker, bundle); - log.error(errorMsg); - throw new IllegalStateException(errorMsg); - } - return CompletableFuture.completedFuture(brokerLookupData); - })); + private CompletableFuture> getBrokerLookupData( + CompletableFuture> owner, + String bundle) { + return owner.thenCompose(broker -> { + if (broker.isEmpty()) { + String errorMsg = String.format( + "Failed to get or assign the owner for bundle:%s", bundle); + log.error(errorMsg); + throw new IllegalStateException(errorMsg); + } + return CompletableFuture.completedFuture(broker.get()); + }).thenCompose(broker -> this.getBrokerRegistry().lookupAsync(broker).thenCompose(brokerLookupData -> { + if (brokerLookupData.isEmpty()) { + String errorMsg = String.format( + "Failed to look up a broker registry:%s for bundle:%s", broker, bundle); + log.error(errorMsg); + throw new IllegalStateException(errorMsg); + } + return CompletableFuture.completedFuture(brokerLookupData); + })); + } + + /** + * Method to get the current owner of the NamespaceBundle + * or set the local broker as the owner if absent. + * + * @param namespaceBundle the NamespaceBundle + * @return The ephemeral node data showing the current ownership info in ServiceUnitStateChannel + */ + public CompletableFuture tryAcquiringOwnership(NamespaceBundle namespaceBundle) { + log.info("Try acquiring ownership for bundle: {} - {}.", namespaceBundle, brokerRegistry.getBrokerId()); + final String bundle = namespaceBundle.toString(); + return dedupeLookupRequest(bundle, k -> { + final CompletableFuture owner = + this.getOwnerAsync(namespaceBundle, bundle, true); + return getBrokerLookupData(owner.thenApply(Optional::ofNullable), bundle); + }).thenApply(brokerLookupData -> { + if (brokerLookupData.isEmpty()) { + throw new IllegalStateException( + "Failed to get the broker lookup data for bundle: " + bundle); + } + return brokerLookupData.get().toNamespaceEphemeralData(); }); + } + + private CompletableFuture> dedupeLookupRequest( + String key, Function>> provider) { + CompletableFuture> future = lookupRequests.computeIfAbsent(key, provider); future.whenComplete((r, t) -> { if (t != null) { assignCounter.incrementFailure(); } - lookupRequests.remove(bundle); + lookupRequests.remove(key); } ); return future; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 99c538e6ecfa3..fbf3534e7d440 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -1192,6 +1192,11 @@ private synchronized void doCleanup(String broker) { log.info("Started ownership cleanup for the inactive broker:{}", broker); int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); + String heartbeatNamespace = + NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()) + .toString(); + String heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), + pulsar.getConfiguration()).toString(); Map orphanSystemServiceUnits = new HashMap<>(); for (var etr : tableview.entrySet()) { @@ -1202,6 +1207,19 @@ private synchronized void doCleanup(String broker) { if (isActiveState(state)) { if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { orphanSystemServiceUnits.put(serviceUnit, stateData); + } else if (serviceUnit.startsWith(heartbeatNamespace) + || serviceUnit.startsWith(heartbeatNamespaceV2)) { + // Skip the heartbeat namespace + log.info("Skip override heartbeat namespace bundle" + + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); + tombstoneAsync(serviceUnit).whenComplete((__, e) -> { + if (e != null) { + log.error("Failed cleaning the heartbeat namespace ownership serviceUnit:{}, " + + "stateData:{}, cleanupErrorCnt:{}.", + serviceUnit, stateData, + totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); + } + }); } else { overrideOwnership(serviceUnit, stateData, broker); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 76b4e093f7dc4..9be8d4938e3e3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -367,7 +367,14 @@ public boolean registerNamespace(NamespaceName nsname, boolean ensureOwned) thro // all pre-registered namespace is assumed to have bundles disabled nsFullBundle = bundleFactory.getFullBundle(nsname); // v2 namespace will always use full bundle object - NamespaceEphemeralData otherData = ownershipCache.tryAcquiringOwnership(nsFullBundle).get(); + final NamespaceEphemeralData otherData; + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { + ExtensibleLoadManagerImpl loadManager = ExtensibleLoadManagerImpl.get(this.loadManager.get()); + otherData = loadManager.tryAcquiringOwnership(nsFullBundle).get(); + } else { + otherData = ownershipCache.tryAcquiringOwnership(nsFullBundle).get(); + } + if (StringUtils.equals(pulsar.getBrokerServiceUrl(), otherData.getNativeUrl()) || StringUtils.equals(pulsar.getBrokerServiceUrlTls(), otherData.getNativeUrlTls())) { if (nsFullBundle != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 22321cc8d35e0..dc844a3d031e9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -89,6 +89,8 @@ import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; import org.apache.pulsar.broker.namespace.NamespaceBundleSplitListener; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.TableViewImpl; @@ -96,6 +98,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.BrokerAssignment; import org.apache.pulsar.common.policies.data.BundlesData; import org.apache.pulsar.common.policies.data.ClusterData; @@ -1038,19 +1041,48 @@ public void testListTopic() throws Exception { } @Test(timeOut = 30 * 1000) - public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws PulsarAdminException { + public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws Exception { + NamespaceName heartbeatNamespacePulsar1V1 = + NamespaceService.getHeartbeatNamespace(pulsar1.getAdvertisedAddress(), pulsar1.getConfiguration()); + NamespaceName heartbeatNamespacePulsar1V2 = + NamespaceService.getHeartbeatNamespaceV2(pulsar1.getAdvertisedAddress(), pulsar1.getConfiguration()); + + NamespaceName heartbeatNamespacePulsar2V1 = + NamespaceService.getHeartbeatNamespace(pulsar2.getAdvertisedAddress(), pulsar2.getConfiguration()); + NamespaceName heartbeatNamespacePulsar2V2 = + NamespaceService.getHeartbeatNamespaceV2(pulsar2.getAdvertisedAddress(), pulsar2.getConfiguration()); + + NamespaceBundle bundle1 = pulsar1.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespacePulsar1V1); + NamespaceBundle bundle2 = pulsar1.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespacePulsar1V2); + + NamespaceBundle bundle3 = pulsar2.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespacePulsar2V1); + NamespaceBundle bundle4 = pulsar2.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespacePulsar2V2); + Set ownedServiceUnitsByPulsar1 = primaryLoadManager.getOwnedServiceUnits(); log.info("Owned service units: {}", ownedServiceUnitsByPulsar1); - assertTrue(ownedServiceUnitsByPulsar1.isEmpty()); + // heartbeat namespace bundle will own by pulsar1 + assertEquals(ownedServiceUnitsByPulsar1.size(), 2); + assertTrue(ownedServiceUnitsByPulsar1.contains(bundle1)); + assertTrue(ownedServiceUnitsByPulsar1.contains(bundle2)); Set ownedServiceUnitsByPulsar2 = secondaryLoadManager.getOwnedServiceUnits(); log.info("Owned service units: {}", ownedServiceUnitsByPulsar2); - assertTrue(ownedServiceUnitsByPulsar2.isEmpty()); + assertEquals(ownedServiceUnitsByPulsar2.size(), 2); + assertTrue(ownedServiceUnitsByPulsar2.contains(bundle3)); + assertTrue(ownedServiceUnitsByPulsar2.contains(bundle4)); Map ownedNamespacesByPulsar1 = admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar1.getLookupServiceAddress()); Map ownedNamespacesByPulsar2 = admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar2.getLookupServiceAddress()); - assertTrue(ownedNamespacesByPulsar1.isEmpty()); - assertTrue(ownedNamespacesByPulsar2.isEmpty()); + assertEquals(ownedNamespacesByPulsar1.size(), 2); + assertTrue(ownedNamespacesByPulsar1.containsKey(bundle1.toString())); + assertTrue(ownedNamespacesByPulsar1.containsKey(bundle2.toString())); + assertEquals(ownedNamespacesByPulsar2.size(), 2); + assertTrue(ownedNamespacesByPulsar2.containsKey(bundle3.toString())); + assertTrue(ownedNamespacesByPulsar2.containsKey(bundle4.toString())); String topic = "persistent://" + defaultTestNamespace + "/test-get-owned-service-units"; admin.topics().createPartitionedTopic(topic, 1); @@ -1083,6 +1115,23 @@ private void assertOwnedServiceUnits( assertEquals(status.broker_assignment, BrokerAssignment.shared); } + @Test(timeOut = 30 * 1000) + public void testTryAcquiringOwnership() + throws PulsarAdminException, ExecutionException, InterruptedException { + final String namespace = "public/testTryAcquiringOwnership"; + admin.namespaces().createNamespace(namespace, 1); + String topic = "persistent://" + namespace + "/test"; + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).get(); + NamespaceEphemeralData namespaceEphemeralData = primaryLoadManager.tryAcquiringOwnership(bundle).get(); + assertEquals(namespaceEphemeralData.getNativeUrl(), pulsar1.getBrokerServiceUrl()); + admin.namespaces().deleteNamespace(namespace, true); + } + + @Test(timeOut = 30 * 1000) + public void testHealthcheck() throws PulsarAdminException { + admin.brokers().healthcheck(TopicVersion.V2); + } + private static abstract class MockBrokerFilter implements BrokerFilter { @Override From d2300e4ea6c628fd90382e55b2bb0b38e9366f24 Mon Sep 17 00:00:00 2001 From: Elliot West Date: Thu, 15 Jun 2023 04:43:24 +0100 Subject: [PATCH 162/494] [fix][admin] Report earliest msg in partitioned backlog (#19465) Co-authored-by: tison --- .../pulsar/broker/admin/AdminApi2Test.java | 16 +++- .../apache/pulsar/client/admin/Topics.java | 2 + .../data/stats/SubscriptionStatsImpl.java | 12 +++ .../policies/data/stats/TopicStatsImpl.java | 12 +++ .../data/stats/SubscriptionStatsImplTest.java | 82 +++++++++++++++++++ .../data/stats/TopicStatsImplTest.java | 81 ++++++++++++++++++ 6 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImplTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index e1800349bbbb7..71892d009313e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.admin; +import static java.util.concurrent.TimeUnit.MINUTES; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.pulsar.broker.BrokerTestUtil.newUniqueName; import static org.mockito.Mockito.spy; @@ -36,6 +37,7 @@ import java.lang.reflect.Field; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.time.Clock; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -1895,6 +1897,8 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti .acknowledgmentGroupTime(0, TimeUnit.SECONDS) .subscribe(); + long start1 = 0; + long start2 = 0; @Cleanup Producer producer = client.newProducer() .topic(topic) @@ -1902,6 +1906,12 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti .create(); for (int i = 0; i < 10; i++) { + if (i == 0) { + start1 = Clock.systemUTC().millis(); + } + if (i == 5) { + start2 = Clock.systemUTC().millis(); + } if (i > 4) { producer.newMessage() .value("message-1".getBytes(StandardCharsets.UTF_8)) @@ -1912,22 +1922,26 @@ public void testBacklogNoDelayedForPartitionedTopic() throws PulsarClientExcepti } } // wait until the message add to delay queue. + long finalStart1 = start1; Awaitility.await().untilAsserted(() -> { TopicStats topicStats = admin.topics().getPartitionedStats(topic, false, true, true, true); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklog(), 10); assertEquals(topicStats.getSubscriptions().get(subName).getBacklogSize(), 440); assertEquals(topicStats.getSubscriptions().get(subName).getMsgBacklogNoDelayed(), 5); + assertTrue(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog() >= finalStart1); }); for (int i = 0; i < 5; i++) { consumer.acknowledge(consumer.receive()); } // Wait the ack send. - Awaitility.await().untilAsserted(() -> { + long finalStart2 = start2; + Awaitility.await().timeout(1, MINUTES).untilAsserted(() -> { TopicStats topicStats2 = admin.topics().getPartitionedStats(topic, false, true, true, true); assertEquals(topicStats2.getSubscriptions().get(subName).getMsgBacklog(), 5); assertEquals(topicStats2.getSubscriptions().get(subName).getBacklogSize(), 223); assertEquals(topicStats2.getSubscriptions().get(subName).getMsgBacklogNoDelayed(), 0); + assertTrue(topicStats2.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog() >= finalStart2); }); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java index f599e2566bffc..156d67e4e58b3 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/Topics.java @@ -1361,6 +1361,8 @@ default PartitionedTopicStats getPartitionedStats(String topic, boolean perParti * Set to true to get precise backlog, Otherwise get imprecise backlog. * @param subscriptionBacklogSize * Whether to get backlog size for each subscription. + * @param getEarliestTimeInBacklog + * Whether to get the earliest time in backlog. * @return a future that can be used to track when the partitioned topic statistics are returned */ CompletableFuture getPartitionedStatsAsync( diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java index fb75fecca3363..d77764e679da0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImpl.java @@ -177,6 +177,7 @@ public void reset() { consumersAfterMarkDeletePosition.clear(); nonContiguousDeletedMessagesRanges = 0; nonContiguousDeletedMessagesRangesSerializedSize = 0; + earliestMsgPublishTimeInBacklog = 0L; delayedMessageIndexSizeInBytes = 0; subscriptionProperties.clear(); filterProcessedMsgCount = 0; @@ -221,6 +222,17 @@ public SubscriptionStatsImpl add(SubscriptionStatsImpl stats) { this.consumersAfterMarkDeletePosition.putAll(stats.consumersAfterMarkDeletePosition); this.nonContiguousDeletedMessagesRanges += stats.nonContiguousDeletedMessagesRanges; this.nonContiguousDeletedMessagesRangesSerializedSize += stats.nonContiguousDeletedMessagesRangesSerializedSize; + if (this.earliestMsgPublishTimeInBacklog != 0 && stats.earliestMsgPublishTimeInBacklog != 0) { + this.earliestMsgPublishTimeInBacklog = Math.min( + this.earliestMsgPublishTimeInBacklog, + stats.earliestMsgPublishTimeInBacklog + ); + } else { + this.earliestMsgPublishTimeInBacklog = Math.max( + this.earliestMsgPublishTimeInBacklog, + stats.earliestMsgPublishTimeInBacklog + ); + } this.delayedMessageIndexSizeInBytes += stats.delayedMessageIndexSizeInBytes; this.subscriptionProperties.putAll(stats.subscriptionProperties); this.filterProcessedMsgCount += stats.filterProcessedMsgCount; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java index ae2438709a017..3bf43dbf41f0a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImpl.java @@ -216,6 +216,7 @@ public void reset() { this.lastOffloadFailureTimeStamp = 0; this.lastOffloadSuccessTimeStamp = 0; this.publishRateLimitedTimes = 0L; + this.earliestMsgPublishTimeInBacklogs = 0L; this.delayedMessageIndexSizeInBytes = 0; this.compaction.reset(); this.ownerBroker = null; @@ -315,6 +316,17 @@ public TopicStatsImpl add(TopicStats ts) { } } } + if (earliestMsgPublishTimeInBacklogs != 0 && ((TopicStatsImpl) ts).earliestMsgPublishTimeInBacklogs != 0) { + earliestMsgPublishTimeInBacklogs = Math.min( + earliestMsgPublishTimeInBacklogs, + ((TopicStatsImpl) ts).earliestMsgPublishTimeInBacklogs + ); + } else { + earliestMsgPublishTimeInBacklogs = Math.max( + earliestMsgPublishTimeInBacklogs, + ((TopicStatsImpl) ts).earliestMsgPublishTimeInBacklogs + ); + } return this; } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java new file mode 100644 index 0000000000000..8a4b5da9edd20 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/SubscriptionStatsImplTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data.stats; + +import static org.testng.Assert.assertEquals; +import org.testng.annotations.Test; + +public class SubscriptionStatsImplTest { + + @Test + public void testReset() { + SubscriptionStatsImpl stats = new SubscriptionStatsImpl(); + stats.earliestMsgPublishTimeInBacklog = 1L; + stats.reset(); + assertEquals(stats.earliestMsgPublishTimeInBacklog, 0L); + + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Earliest() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.earliestMsgPublishTimeInBacklog = 10L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.earliestMsgPublishTimeInBacklog = 20L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 10L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_First0() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.earliestMsgPublishTimeInBacklog = 0L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.earliestMsgPublishTimeInBacklog = 20L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 20L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Second0() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.earliestMsgPublishTimeInBacklog = 10L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.earliestMsgPublishTimeInBacklog = 0L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 10L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Zero() { + SubscriptionStatsImpl stats1 = new SubscriptionStatsImpl(); + stats1.earliestMsgPublishTimeInBacklog = 0L; + + SubscriptionStatsImpl stats2 = new SubscriptionStatsImpl(); + stats2.earliestMsgPublishTimeInBacklog = 0L; + + SubscriptionStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklog, 0L); + } +} \ No newline at end of file diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImplTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImplTest.java new file mode 100644 index 0000000000000..09cef4c4d0f82 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/stats/TopicStatsImplTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.policies.data.stats; + +import static org.testng.Assert.assertEquals; +import org.testng.annotations.Test; + +public class TopicStatsImplTest { + + @Test + public void testReset() { + TopicStatsImpl stats = new TopicStatsImpl(); + stats.earliestMsgPublishTimeInBacklogs = 1L; + stats.reset(); + assertEquals(stats.earliestMsgPublishTimeInBacklogs, 0L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Earliest() { + TopicStatsImpl stats1 = new TopicStatsImpl(); + stats1.earliestMsgPublishTimeInBacklogs = 10L; + + TopicStatsImpl stats2 = new TopicStatsImpl(); + stats2.earliestMsgPublishTimeInBacklogs = 20L; + + TopicStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklogs, 10L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_First0() { + TopicStatsImpl stats1 = new TopicStatsImpl(); + stats1.earliestMsgPublishTimeInBacklogs = 0L; + + TopicStatsImpl stats2 = new TopicStatsImpl(); + stats2.earliestMsgPublishTimeInBacklogs = 20L; + + TopicStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklogs, 20L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Second0() { + TopicStatsImpl stats1 = new TopicStatsImpl(); + stats1.earliestMsgPublishTimeInBacklogs = 10L; + + TopicStatsImpl stats2 = new TopicStatsImpl(); + stats2.earliestMsgPublishTimeInBacklogs = 0L; + + TopicStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklogs, 10L); + } + + @Test + public void testAdd_EarliestMsgPublishTimeInBacklogs_Zero() { + TopicStatsImpl stats1 = new TopicStatsImpl(); + stats1.earliestMsgPublishTimeInBacklogs = 0L; + + TopicStatsImpl stats2 = new TopicStatsImpl(); + stats2.earliestMsgPublishTimeInBacklogs = 0L; + + TopicStatsImpl aggregate = stats1.add(stats2); + assertEquals(aggregate.earliestMsgPublishTimeInBacklogs, 0L); + } +} \ No newline at end of file From 083b158e90643a98d08352103bb1e568aabd5ead Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 15 Jun 2023 12:48:40 +0800 Subject: [PATCH 163/494] [fix][broker] release orphan replicator after topic closed (#20567) Motivation: When the `replicator.producer` retries to start[1] the topic close[2] executed concurrently, there is an orphan replicator after this topic is closed. Modifications: If the topic was already closed, stop to retry. --- .../broker/service/AbstractReplicator.java | 38 ++++- .../NonPersistentReplicator.java | 2 +- .../persistent/PersistentReplicator.java | 2 +- .../service/AbstractReplicatorTest.java | 145 ++++++++++++++++++ 4 files changed, 181 insertions(+), 6 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index deab89cda72dc..539f178d665cc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; @@ -47,6 +48,7 @@ public abstract class AbstractReplicator { protected final PulsarClientImpl replicationClient; protected final PulsarClientImpl client; protected String replicatorId; + protected final Topic localTopic; protected volatile ProducerImpl producer; public static final String REPL_PRODUCER_NAME_DELIMITER = "-->"; @@ -67,11 +69,12 @@ protected enum State { Stopped, Starting, Started, Stopping } - public AbstractReplicator(String localCluster, String localTopicName, String remoteCluster, String remoteTopicName, + public AbstractReplicator(String localCluster, Topic localTopic, String remoteCluster, String remoteTopicName, String replicatorPrefix, BrokerService brokerService, PulsarClientImpl replicationClient) throws PulsarServerException { this.brokerService = brokerService; - this.localTopicName = localTopicName; + this.localTopic = localTopic; + this.localTopicName = localTopic.getName(); this.replicatorPrefix = replicatorPrefix; this.localCluster = localCluster.intern(); this.remoteTopicName = remoteTopicName; @@ -120,7 +123,8 @@ public synchronized void startProducer() { replicatorId, waitTimeMs / 1000.0); } // BackOff before retrying - brokerService.executor().schedule(this::startProducer, waitTimeMs, TimeUnit.MILLISECONDS); + brokerService.executor().schedule(this::checkTopicActiveAndRetryStartProducer, waitTimeMs, + TimeUnit.MILLISECONDS); return; } State state = STATE_UPDATER.get(this); @@ -147,7 +151,8 @@ public synchronized void startProducer() { replicatorId, ex.getMessage(), waitTimeMs / 1000.0); // BackOff before retrying - brokerService.executor().schedule(this::startProducer, waitTimeMs, TimeUnit.MILLISECONDS); + brokerService.executor().schedule(this::checkTopicActiveAndRetryStartProducer, waitTimeMs, + TimeUnit.MILLISECONDS); } else { log.warn("[{}] Failed to create remote producer. Replicator state: {}", replicatorId, STATE_UPDATER.get(this), ex); @@ -157,6 +162,31 @@ public synchronized void startProducer() { } + protected void checkTopicActiveAndRetryStartProducer() { + isLocalTopicActive().thenAccept(isTopicActive -> { + if (isTopicActive) { + startProducer(); + } + }).exceptionally(ex -> { + log.warn("[{}] Stop retry to create producer due to topic load fail. Replicator state: {}", replicatorId, + STATE_UPDATER.get(this), ex); + return null; + }); + } + + protected CompletableFuture isLocalTopicActive() { + CompletableFuture> topicFuture = brokerService.getTopics().get(localTopicName); + if (topicFuture == null){ + return CompletableFuture.completedFuture(false); + } + return topicFuture.thenApplyAsync(optional -> { + if (optional.isEmpty()) { + return false; + } + return optional.get() == localTopic; + }, brokerService.executor()); + } + protected synchronized CompletableFuture closeProducerAsync() { if (producer == null) { STATE_UPDATER.set(this, State.Stopped); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java index 514db4219db98..087c5f932008f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentReplicator.java @@ -50,7 +50,7 @@ public class NonPersistentReplicator extends AbstractReplicator implements Repli public NonPersistentReplicator(NonPersistentTopic topic, String localCluster, String remoteCluster, BrokerService brokerService, PulsarClientImpl replicationClient) throws PulsarServerException { - super(localCluster, topic.getName(), remoteCluster, topic.getName(), topic.getReplicatorPrefix(), brokerService, + super(localCluster, topic, remoteCluster, topic.getName(), topic.getReplicatorPrefix(), brokerService, replicationClient); producerBuilder.blockIfQueueFull(false); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index d882cbf56b2e8..ccf70eecec35c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -112,7 +112,7 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man String remoteCluster, String remoteTopic, BrokerService brokerService, PulsarClientImpl replicationClient) throws PulsarServerException { - super(localCluster, localTopic.getName(), remoteCluster, remoteTopic, localTopic.getReplicatorPrefix(), + super(localCluster, localTopic, remoteCluster, remoteTopic, localTopic.getReplicatorPrefix(), brokerService, replicationClient); this.topic = localTopic; this.cursor = cursor; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java new file mode 100644 index 0000000000000..294a9b341ec69 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyBoolean; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import io.netty.channel.DefaultEventLoop; +import io.netty.util.internal.DefaultPriorityQueue; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.Assert; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class AbstractReplicatorTest { + + @Test + public void testRetryStartProducerStoppedByTopicRemove() throws Exception { + final String localCluster = "localCluster"; + final String remoteCluster = "remoteCluster"; + final String topicName = "remoteTopicName"; + final String replicatorPrefix = "pulsar.repl"; + final DefaultEventLoop eventLoopGroup = new DefaultEventLoop(); + // Mock services. + final ServiceConfiguration pulsarConfig = mock(ServiceConfiguration.class); + final PulsarService pulsar = mock(PulsarService.class); + final BrokerService broker = mock(BrokerService.class); + final Topic localTopic = mock(Topic.class); + final PulsarClientImpl localClient = mock(PulsarClientImpl.class); + final PulsarClientImpl remoteClient = mock(PulsarClientImpl.class); + final ProducerBuilder producerBuilder = mock(ProducerBuilder.class); + final ConcurrentOpenHashMap>> topics = new ConcurrentOpenHashMap<>(); + when(broker.executor()).thenReturn(eventLoopGroup); + when(broker.getTopics()).thenReturn(topics); + when(remoteClient.newProducer(any(Schema.class))).thenReturn(producerBuilder); + when(broker.pulsar()).thenReturn(pulsar); + when(pulsar.getClient()).thenReturn(localClient); + when(pulsar.getConfiguration()).thenReturn(pulsarConfig); + when(pulsarConfig.getReplicationProducerQueueSize()).thenReturn(100); + when(localTopic.getName()).thenReturn(topicName); + when(producerBuilder.topic(any())).thenReturn(producerBuilder); + when(producerBuilder.messageRoutingMode(any())).thenReturn(producerBuilder); + when(producerBuilder.enableBatching(anyBoolean())).thenReturn(producerBuilder); + when(producerBuilder.sendTimeout(anyInt(), any())).thenReturn(producerBuilder); + when(producerBuilder.maxPendingMessages(anyInt())).thenReturn(producerBuilder); + when(producerBuilder.producerName(anyString())).thenReturn(producerBuilder); + // Mock create producer fail. + when(producerBuilder.create()).thenThrow(new RuntimeException("mocked ex")); + when(producerBuilder.createAsync()) + .thenReturn(CompletableFuture.failedFuture(new RuntimeException("mocked ex"))); + // Make race condition: "retry start producer" and "close replicator". + final ReplicatorInTest replicator = new ReplicatorInTest(localCluster, localTopic, remoteCluster, topicName, + replicatorPrefix, broker, remoteClient); + replicator.startProducer(); + replicator.disconnect(); + + // Verify task will done. + Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + AtomicInteger taskCounter = new AtomicInteger(); + CountDownLatch checkTaskFinished = new CountDownLatch(1); + eventLoopGroup.execute(() -> { + synchronized (replicator) { + LinkedBlockingQueue taskQueue = WhiteboxImpl.getInternalState(eventLoopGroup, "taskQueue"); + DefaultPriorityQueue scheduledTaskQueue = + WhiteboxImpl.getInternalState(eventLoopGroup, "scheduledTaskQueue"); + taskCounter.set(taskQueue.size() + scheduledTaskQueue.size()); + checkTaskFinished.countDown(); + } + }); + checkTaskFinished.await(); + Assert.assertEquals(taskCounter.get(), 0); + }); + } + + private static class ReplicatorInTest extends AbstractReplicator { + + public ReplicatorInTest(String localCluster, Topic localTopic, String remoteCluster, String remoteTopicName, + String replicatorPrefix, BrokerService brokerService, + PulsarClientImpl replicationClient) throws PulsarServerException { + super(localCluster, localTopic, remoteCluster, remoteTopicName, replicatorPrefix, brokerService, + replicationClient); + } + + @Override + protected String getProducerName() { + return "pulsar.repl.producer"; + } + + @Override + protected void readEntries(Producer producer) { + + } + + @Override + protected Position getReplicatorReadPosition() { + return PositionImpl.EARLIEST; + } + + @Override + protected long getNumberOfEntriesInBacklog() { + return 0; + } + + @Override + protected void disableReplicatorRead() { + + } + } +} From c3cae36a6ff8a378e622da49c1d2453fc0e05d25 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Wed, 21 Jun 2023 13:27:16 +0800 Subject: [PATCH 164/494] [fix][broker] getOwnedServiceUnits NPE (#20625) --- .../loadbalance/extensions/ExtensibleLoadManagerImpl.java | 6 +++++- .../extensions/ExtensibleLoadManagerImplTest.java | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index a9a6e6138bda9..d0c55a6519159 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -169,7 +169,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private SplitManager splitManager; - private boolean started = false; + private volatile boolean started = false; private final AssignCounter assignCounter = new AssignCounter(); @Getter @@ -191,6 +191,10 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { * Get all the bundles that are owned by this broker. */ public Set getOwnedServiceUnits() { + if (!started) { + log.warn("Failed to get owned service units, load manager is not started."); + return Collections.emptySet(); + } Set> entrySet = serviceUnitStateChannel.getOwnershipEntrySet(); String brokerId = brokerRegistry.getBrokerId(); return entrySet.stream() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index dc844a3d031e9..7af10c4aadcc5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -1115,6 +1115,14 @@ private void assertOwnedServiceUnits( assertEquals(status.broker_assignment, BrokerAssignment.shared); } + @Test(timeOut = 30 * 1000) + public void testGetOwnedServiceUnitsWhenLoadManagerNotStart() { + ExtensibleLoadManagerImpl loadManager = new ExtensibleLoadManagerImpl(); + Set ownedServiceUnits = loadManager.getOwnedServiceUnits(); + assertNotNull(ownedServiceUnits); + assertTrue(ownedServiceUnits.isEmpty()); + } + @Test(timeOut = 30 * 1000) public void testTryAcquiringOwnership() throws PulsarAdminException, ExecutionException, InterruptedException { From fcf5d29c2010570a7e12ee233d58dfbcfc6fea93 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sat, 29 Apr 2023 01:52:52 +0800 Subject: [PATCH 165/494] [improve][broker] Support cgroup v2 by using `jdk.internal.platform.Metrics` in Pulsar Loadbalancer (#16832) (cherry picked from commit b8543ad979798d89da9f35a9375de7e16b7e5e25) --- bin/pulsar | 2 + buildtools/pom.xml | 1 + pom.xml | 1 + .../broker/loadbalance/LinuxInfoUtils.java | 66 ++++++++++++++++--- .../impl/LinuxBrokerHostUsageImpl.java | 2 +- .../loadbalance/SimpleBrokerStartTest.java | 26 ++++++++ .../impl/LinuxBrokerHostUsageImplTest.java | 37 ++++++++++- 7 files changed, 122 insertions(+), 13 deletions(-) diff --git a/bin/pulsar b/bin/pulsar index 9d924bb296c59..20ed1f7f22b0f 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -307,6 +307,8 @@ if [[ -z "$IS_JAVA_8" ]]; then OPTS="$OPTS --add-opens java.management/sun.management=ALL-UNNAMED" # MBeanStatsGenerator OPTS="$OPTS --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED" + # LinuxInfoUtils + OPTS="$OPTS --add-opens java.base/jdk.internal.platform=ALL-UNNAMED" fi OPTS="-cp $PULSAR_CLASSPATH $OPTS" diff --git a/buildtools/pom.xml b/buildtools/pom.xml index cbc4dfee05e7a..1a02938fc531a 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -57,6 +57,7 @@ --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/jdk.internal.platform=ALL-UNNAMED diff --git a/pom.xml b/pom.xml index c1e6454eb6682..6806efce101c7 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,7 @@ flexible messaging model and an intuitive client API. --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.management/sun.management=ALL-UNNAMED --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED + --add-opens java.base/jdk.internal.platform=ALL-UNNAMED true 4 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index c9fce46a30515..17aa7170fc63c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -18,7 +18,9 @@ */ package org.apache.pulsar.broker.loadbalance; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; +import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -45,6 +47,7 @@ public class LinuxInfoUtils { private static final String CGROUPS_CPU_USAGE_PATH = "/sys/fs/cgroup/cpu/cpuacct.usage"; private static final String CGROUPS_CPU_LIMIT_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"; private static final String CGROUPS_CPU_LIMIT_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"; + // proc states private static final String PROC_STAT_PATH = "/proc/stat"; private static final String NIC_PATH = "/sys/class/net/"; @@ -52,6 +55,30 @@ public class LinuxInfoUtils { private static final int ARPHRD_ETHER = 1; private static final String NIC_SPEED_TEMPLATE = "/sys/class/net/%s/speed"; + private static Object /*jdk.internal.platform.Metrics*/ metrics; + private static Method getMetricsProviderMethod; + private static Method getCpuQuotaMethod; + private static Method getCpuPeriodMethod; + private static Method getCpuUsageMethod; + + static { + try { + metrics = Class.forName("jdk.internal.platform.Container").getMethod("metrics") + .invoke(null); + if (metrics != null) { + getMetricsProviderMethod = metrics.getClass().getMethod("getProvider"); + getMetricsProviderMethod.setAccessible(true); + getCpuQuotaMethod = metrics.getClass().getMethod("getCpuQuota"); + getCpuQuotaMethod.setAccessible(true); + getCpuPeriodMethod = metrics.getClass().getMethod("getCpuPeriod"); + getCpuPeriodMethod.setAccessible(true); + getCpuUsageMethod = metrics.getClass().getMethod("getCpuUsage"); + getCpuUsageMethod.setAccessible(true); + } + } catch (Throwable e) { + log.warn("Failed to get runtime metrics", e); + } + } /** * Determine whether the OS is the linux kernel. @@ -66,9 +93,14 @@ public static boolean isLinux() { */ public static boolean isCGroupEnabled() { try { - return Files.exists(Paths.get(CGROUPS_CPU_USAGE_PATH)); + if (metrics == null) { + return Files.exists(Paths.get(CGROUPS_CPU_USAGE_PATH)); + } + String provider = (String) getMetricsProviderMethod.invoke(metrics); + log.info("[LinuxInfo] The system metrics provider is: {}", provider); + return provider.contains("cgroup"); } catch (Exception e) { - log.warn("[LinuxInfo] Failed to check cgroup CPU usage file: {}", e.getMessage()); + log.warn("[LinuxInfo] Failed to check cgroup CPU: {}", e.getMessage()); return false; } } @@ -81,13 +113,21 @@ public static boolean isCGroupEnabled() { public static double getTotalCpuLimit(boolean isCGroupsEnabled) { if (isCGroupsEnabled) { try { - long quota = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_QUOTA_PATH)); - long period = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_PERIOD_PATH)); + long quota; + long period; + if (metrics != null && getCpuQuotaMethod != null && getCpuPeriodMethod != null) { + quota = (long) getCpuQuotaMethod.invoke(metrics); + period = (long) getCpuPeriodMethod.invoke(metrics); + } else { + quota = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_QUOTA_PATH)); + period = readLongFromFile(Paths.get(CGROUPS_CPU_LIMIT_PERIOD_PATH)); + } + if (quota > 0) { return 100.0 * quota / period; } - } catch (IOException e) { - log.warn("[LinuxInfo] Failed to read CPU quotas from cgroups", e); + } catch (Exception e) { + log.warn("[LinuxInfo] Failed to read CPU quotas from cgroup", e); // Fallback to availableProcessors } } @@ -99,11 +139,14 @@ public static double getTotalCpuLimit(boolean isCGroupsEnabled) { * Get CGroup cpu usage. * @return Cpu usage */ - public static double getCpuUsageForCGroup() { + public static long getCpuUsageForCGroup() { try { + if (metrics != null && getCpuUsageMethod != null) { + return (long) getCpuUsageMethod.invoke(metrics); + } return readLongFromFile(Paths.get(CGROUPS_CPU_USAGE_PATH)); - } catch (IOException e) { - log.error("[LinuxInfo] Failed to read CPU usage from {}", CGROUPS_CPU_USAGE_PATH, e); + } catch (Exception e) { + log.error("[LinuxInfo] Failed to read CPU usage from cgroup", e); return -1; } } @@ -291,6 +334,11 @@ enum Operstate { UP } + @VisibleForTesting + public static Object getMetrics() { + return metrics; + } + @AllArgsConstructor public enum NICUsageType { // transport diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 8412658d86ae8..2f7ca614943b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -140,7 +140,7 @@ private double getTotalCpuUsage(double elapsedTimeSeconds) { } private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) { - double usage = getCpuUsageForCGroup(); + double usage = (double) getCpuUsageForCGroup(); double currentUsage = usage - lastCpuUsage; lastCpuUsage = usage; return 100 * currentUsage / elapsedTimeSeconds / TimeUnit.SECONDS.toNanos(1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index 6de2eb90f12ef..28dde8b7f559d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -20,6 +20,8 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Optional; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; @@ -28,6 +30,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.testng.Assert; import org.testng.annotations.Test; @Slf4j @@ -96,4 +99,27 @@ public void testNoNICSpeed() throws Exception { } + @Test + public void testCGroupMetrics() { + if (!LinuxInfoUtils.isLinux()) { + return; + } + + boolean existsCGroup = Files.exists(Paths.get("/sys/fs/cgroup")); + boolean cGroupEnabled = LinuxInfoUtils.isCGroupEnabled(); + Assert.assertEquals(cGroupEnabled, existsCGroup); + + double totalCpuLimit = LinuxInfoUtils.getTotalCpuLimit(cGroupEnabled); + log.info("totalCpuLimit: {}", totalCpuLimit); + Assert.assertTrue(totalCpuLimit > 0.0); + + if (cGroupEnabled) { + Assert.assertNotNull(LinuxInfoUtils.getMetrics()); + + long cpuUsageForCGroup = LinuxInfoUtils.getCpuUsageForCGroup(); + log.info("cpuUsageForCGroup: {}", cpuUsageForCGroup); + Assert.assertTrue(cpuUsageForCGroup > 0); + } + } + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java index 39298dce0b16a..563f707c445b2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImplTest.java @@ -18,15 +18,19 @@ */ package org.apache.pulsar.broker.loadbalance.impl; -import lombok.Cleanup; -import org.testng.Assert; -import org.testng.annotations.Test; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.loadbalance.LinuxInfoUtils; +import org.testng.Assert; +import org.testng.annotations.Test; +@Slf4j public class LinuxBrokerHostUsageImplTest { @Test @@ -42,4 +46,31 @@ public void checkOverrideBrokerNicSpeedGbps() { double totalLimit = linuxBrokerHostUsage.getTotalNicLimitWithConfiguration(nics); Assert.assertEquals(totalLimit, 3.0 * 1000 * 1000 * 3); } + + @Test + public void testCpuUsage() throws InterruptedException { + if (!LinuxInfoUtils.isLinux()) { + return; + } + + @Cleanup("shutdown") + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + LinuxBrokerHostUsageImpl linuxBrokerHostUsage = + new LinuxBrokerHostUsageImpl(Integer.MAX_VALUE, Optional.empty(), executorService); + + linuxBrokerHostUsage.calculateBrokerHostUsage(); + TimeUnit.SECONDS.sleep(1); + linuxBrokerHostUsage.calculateBrokerHostUsage(); + + double usage = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().usage; + double limit = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().limit; + float percentUsage = linuxBrokerHostUsage.getBrokerHostUsage().getCpu().percentUsage(); + + Assert.assertTrue(usage > 0); + Assert.assertTrue(limit > 0); + Assert.assertTrue(limit >= usage); + Assert.assertTrue(percentUsage > 0); + + log.info("usage: {}, limit: {}, percentUsage: {}", usage, limit, percentUsage); + } } From 945707a338ecff219189c67524a0aa61df529236 Mon Sep 17 00:00:00 2001 From: Massimiliano Mirelli Date: Wed, 24 May 2023 12:20:42 +0200 Subject: [PATCH 166/494] [improve][bk] Add integration test with bookie http server enabled (#20149) Signed-off-by: tison Co-authored-by: tison (cherry picked from commit 3f2978d32223d61f04db1de330f5b167a63925ae) --- ...eeperInstallWithHttpServerEnabledTest.java | 84 +++++++++++++++++++ .../integration/topologies/PulsarCluster.java | 36 +++++--- .../topologies/PulsarClusterSpec.java | 10 +++ .../topologies/PulsarClusterTestBase.java | 2 + 4 files changed, 120 insertions(+), 12 deletions(-) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java new file mode 100644 index 0000000000000..03d7f974ab39b --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/bookkeeper/BookkeeperInstallWithHttpServerEnabledTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.bookkeeper; + +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.tests.integration.docker.ContainerExecResult; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterTestBase; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; +import static org.testng.Assert.assertEquals; + +/** + * Test bookkeeper setup with http server enabled. + */ +@Slf4j +public class BookkeeperInstallWithHttpServerEnabledTest extends PulsarClusterTestBase { + + @BeforeClass(alwaysRun = true) + @Override + public final void setupCluster() throws Exception { + incrementSetupNumber(); + + final String clusterName = Stream.of(this.getClass().getSimpleName(), randomName(5)) + .filter(s -> !s.isEmpty()) + .collect(joining("-")); + bookkeeperEnvs.put("httpServerEnabled", "true"); + bookieAdditionalPorts.add(8000); + PulsarClusterSpec spec = PulsarClusterSpec.builder() + .numBookies(2) + .numBrokers(1) + .bookkeeperEnvs(bookkeeperEnvs) + .bookieAdditionalPorts(bookieAdditionalPorts) + .clusterName(clusterName) + .build(); + + log.info("Setting up cluster {} with {} bookies, {} brokers", + spec.clusterName(), spec.numBookies(), spec.numBrokers()); + + pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + + log.info("Cluster {} is setup", spec.clusterName()); + } + + @AfterClass(alwaysRun = true) + @Override + public final void tearDownCluster() throws Exception { + super.tearDownCluster(); + } + + @Test + public void testBookieHttpServerIsRunning() throws Exception { + ContainerExecResult result = pulsarCluster.getAnyBookie().execCmd( + PulsarCluster.CURL, + "-X", + "GET", + "http://localhost:8000/heartbeat"); + assertEquals(result.getExitCode(), 0); + assertEquals(result.getStdout(), "OK\n"); + } +} diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index fcc0feec6d44f..bd11b7d387383 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -157,18 +157,26 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s // create bookies bookieContainers.putAll( - runNumContainers("bookie", spec.numBookies(), (name) -> new BKContainer(clusterName, name) - .withNetwork(network) - .withNetworkAliases(appendClusterName(name)) - .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) - .withEnv("useHostNameAsBookieID", "true") - // Disable fsyncs for tests since they're slow within the containers - .withEnv("journalSyncData", "false") - .withEnv("journalMaxGroupWaitMSec", "0") - .withEnv("clusterName", clusterName) - .withEnv("diskUsageThreshold", "0.99") - .withEnv("nettyMaxFrameSizeBytes", "" + spec.maxMessageSize) - ) + runNumContainers("bookie", spec.numBookies(), (name) -> { + BKContainer bookieContainer = new BKContainer(clusterName, name) + .withNetwork(network) + .withNetworkAliases(appendClusterName(name)) + .withEnv("zkServers", appendClusterName(ZKContainer.NAME)) + .withEnv("useHostNameAsBookieID", "true") + // Disable fsyncs for tests since they're slow within the containers + .withEnv("journalSyncData", "false") + .withEnv("journalMaxGroupWaitMSec", "0") + .withEnv("clusterName", clusterName) + .withEnv("diskUsageThreshold", "0.99") + .withEnv("nettyMaxFrameSizeBytes", String.valueOf(spec.maxMessageSize)); + if (spec.bookkeeperEnvs != null) { + bookieContainer.withEnv(spec.bookkeeperEnvs); + } + if (spec.bookieAdditionalPorts != null) { + spec.bookieAdditionalPorts.forEach(bookieContainer::addExposedPort); + } + return bookieContainer; + }) ); // create brokers @@ -740,4 +748,8 @@ public void dumpFunctionLogs(String name) { private String appendClusterName(String name) { return sharedCsContainer ? clusterName + "-" + name : name; } + + public BKContainer getAnyBookie() { + return getAnyContainer(bookieContainers, "bookie"); + } } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java index 385af99a6644b..fa28d20e6b356 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterSpec.java @@ -150,6 +150,11 @@ public class PulsarClusterSpec { */ Map brokerEnvs; + /** + * Specify envs for bookkeeper. + */ + Map bookkeeperEnvs; + /** * Specify mount files. */ @@ -167,4 +172,9 @@ public class PulsarClusterSpec { * Additional ports to expose on broker containers. */ List brokerAdditionalPorts; + + /** + * Additional ports to expose on bookie containers. + */ + List bookieAdditionalPorts; } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java index d7a1906ec582e..ae9e44fa98254 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarClusterTestBase.java @@ -34,8 +34,10 @@ @Slf4j public abstract class PulsarClusterTestBase extends PulsarTestBase { protected final Map brokerEnvs = new HashMap<>(); + protected final Map bookkeeperEnvs = new HashMap<>(); protected final Map proxyEnvs = new HashMap<>(); protected final List brokerAdditionalPorts = new LinkedList<>(); + protected final List bookieAdditionalPorts = new LinkedList<>(); @Override protected final void setup() throws Exception { From 5db3258eb04b550bc3f992daf7f37d01a465f43b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 28 Jun 2023 21:10:05 +0300 Subject: [PATCH 167/494] [improve][test] Disable disk usage threshold & geoip download and enable logging for Elastic Testcontainers (#20671) (cherry picked from commit e97fe5b8ac376ecead0097b184f0f1436fce88e4) --- .../elasticsearch/ElasticSearchTestBase.java | 32 ++++++++++----- .../io/sinks/ElasticSearch7SinkTester.java | 4 +- .../io/sinks/ElasticSearch8SinkTester.java | 5 +-- .../io/sinks/ElasticSearchSinkTester.java | 40 ++++++++++++++----- .../io/sinks/OpenSearchSinkTester.java | 13 +++--- .../integration/topologies/PulsarCluster.java | 2 + 6 files changed, 66 insertions(+), 30 deletions(-) diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java index 4c6fd020fa338..0f5a42051c7d1 100644 --- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java +++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchTestBase.java @@ -18,10 +18,6 @@ */ package org.apache.pulsar.io.elasticsearch; -import java.io.IOException; -import java.util.Map; -import java.util.Optional; - import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch.security.CreateApiKeyRequest; import co.elastic.clients.elasticsearch.security.CreateApiKeyResponse; @@ -29,6 +25,10 @@ import co.elastic.clients.elasticsearch.security.GetTokenResponse; import co.elastic.clients.elasticsearch.security.get_token.AccessTokenGrantType; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.io.elasticsearch.client.elastic.ElasticSearchJavaRestClient; import org.apache.pulsar.io.elasticsearch.client.opensearch.OpenSearchHighLevelRestClient; import org.opensearch.client.Request; @@ -36,10 +36,11 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; +@Slf4j public abstract class ElasticSearchTestBase { public static final String ELASTICSEARCH_8 = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE_V8")) - .orElse("docker.elastic.co/elasticsearch/elasticsearch:8.5.1"); + .orElse("docker.elastic.co/elasticsearch/elasticsearch:8.5.3"); public static final String ELASTICSEARCH_7 = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE_V7")) .orElse("docker.elastic.co/elasticsearch/elasticsearch:7.17.7"); @@ -54,17 +55,28 @@ public ElasticSearchTestBase(String elasticImageName) { } protected ElasticsearchContainer createElasticsearchContainer() { + ElasticsearchContainer elasticsearchContainer; if (elasticImageName.equals(OPENSEARCH)) { DockerImageName dockerImageName = DockerImageName.parse(OPENSEARCH).asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"); - return new ElasticsearchContainer(dockerImageName) + elasticsearchContainer = new ElasticsearchContainer(dockerImageName) .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms128m -Xmx256m") .withEnv("bootstrap.memory_lock", "true") .withEnv("plugins.security.disabled", "true"); + } else { + elasticsearchContainer = new ElasticsearchContainer(elasticImageName) + .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m") + .withEnv("xpack.security.enabled", "false") + .withEnv("xpack.security.http.ssl.enabled", "false"); + } + configureElasticContainer(elasticsearchContainer); + return elasticsearchContainer; + } + + protected void configureElasticContainer(ElasticsearchContainer elasticContainer) { + if (getCompatibilityMode() != ElasticSearchConfig.CompatibilityMode.OPENSEARCH) { + elasticContainer.withEnv("ingest.geoip.downloader.enabled", "false"); } - return new ElasticsearchContainer(elasticImageName) - .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m") - .withEnv("xpack.security.enabled", "false") - .withEnv("xpack.security.http.ssl.enabled", "false"); + elasticContainer.withLogConsumer(o -> log.info("elastic> {}", o.getUtf8String())); } protected ElasticSearchConfig.CompatibilityMode getCompatibilityMode() { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch7SinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch7SinkTester.java index 65b38c677bfc5..d99fcad252706 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch7SinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch7SinkTester.java @@ -19,7 +19,6 @@ package org.apache.pulsar.tests.integration.io.sinks; import java.util.Optional; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testcontainers.elasticsearch.ElasticsearchContainer; public class ElasticSearch7SinkTester extends ElasticSearchSinkTester { @@ -32,8 +31,9 @@ public ElasticSearch7SinkTester(boolean schemaEnable) { super(schemaEnable); } + @Override - protected ElasticsearchContainer createSinkService(PulsarCluster cluster) { + protected ElasticsearchContainer createElasticContainer() { return new ElasticsearchContainer(ELASTICSEARCH_7) .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m"); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch8SinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch8SinkTester.java index bb52c4ff03fea..8e7617a82a5b9 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch8SinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearch8SinkTester.java @@ -19,13 +19,12 @@ package org.apache.pulsar.tests.integration.io.sinks; import java.util.Optional; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.testcontainers.elasticsearch.ElasticsearchContainer; public class ElasticSearch8SinkTester extends ElasticSearchSinkTester { public static final String ELASTICSEARCH_8 = Optional.ofNullable(System.getenv("ELASTICSEARCH_IMAGE_V8")) - .orElse("docker.elastic.co/elasticsearch/elasticsearch:8.5.1"); + .orElse("docker.elastic.co/elasticsearch/elasticsearch:8.5.3"); public ElasticSearch8SinkTester(boolean schemaEnable) { @@ -33,7 +32,7 @@ public ElasticSearch8SinkTester(boolean schemaEnable) { } @Override - protected ElasticsearchContainer createSinkService(PulsarCluster cluster) { + protected ElasticsearchContainer createElasticContainer() { return new ElasticsearchContainer(ELASTICSEARCH_8) .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx256m") .withEnv("xpack.security.enabled", "false") diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java index 546dd1b9113ab..0784055d29003 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/ElasticSearchSinkTester.java @@ -19,15 +19,6 @@ package org.apache.pulsar.tests.integration.io.sinks; import static org.testng.Assert.assertTrue; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; @@ -35,6 +26,13 @@ import co.elastic.clients.transport.ElasticsearchTransport; import co.elastic.clients.transport.rest_client.RestClientTransport; import com.google.common.collect.ImmutableMap; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import lombok.AllArgsConstructor; import lombok.Cleanup; import lombok.Data; @@ -46,6 +44,7 @@ import org.apache.pulsar.common.schema.KeyValue; import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.awaitility.Awaitility; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; @@ -100,6 +99,29 @@ public ElasticSearchSinkTester(boolean schemaEnable) { } } + @Override + protected final ElasticsearchContainer createSinkService(PulsarCluster cluster) { + ElasticsearchContainer elasticContainer = createElasticContainer(); + configureElasticContainer(elasticContainer); + return elasticContainer; + } + + protected void configureElasticContainer(ElasticsearchContainer elasticContainer) { + if (!isOpenSearch()) { + elasticContainer.withEnv("ingest.geoip.downloader.enabled", "false"); + } + + // allow disk to fill up beyond default 90% threshold + elasticContainer.withEnv("cluster.routing.allocation.disk.threshold_enabled", "false"); + + elasticContainer.withLogConsumer(o -> log.info("elastic> {}", o.getUtf8String())); + } + + protected boolean isOpenSearch() { + return false; + } + + protected abstract ElasticsearchContainer createElasticContainer(); @Override public void prepareSink() throws Exception { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/OpenSearchSinkTester.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/OpenSearchSinkTester.java index 1e10cc4189c1a..75f0fdac6f90c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/OpenSearchSinkTester.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/io/sinks/OpenSearchSinkTester.java @@ -18,9 +18,10 @@ */ package org.apache.pulsar.tests.integration.io.sinks; +import static org.testng.Assert.assertTrue; +import java.util.Map; import java.util.Optional; import org.apache.http.HttpHost; -import org.apache.pulsar.tests.integration.topologies.PulsarCluster; import org.awaitility.Awaitility; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; @@ -31,10 +32,6 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.utility.DockerImageName; -import java.util.Map; - -import static org.testng.Assert.assertTrue; - public class OpenSearchSinkTester extends ElasticSearchSinkTester { public static final String OPENSEARCH = Optional.ofNullable(System.getenv("OPENSEARCH_IMAGE")) @@ -48,7 +45,7 @@ public OpenSearchSinkTester(boolean schemaEnable) { } @Override - protected ElasticsearchContainer createSinkService(PulsarCluster cluster) { + protected ElasticsearchContainer createElasticContainer() { DockerImageName dockerImageName = DockerImageName.parse(OPENSEARCH) .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch"); return new ElasticsearchContainer(dockerImageName) @@ -57,6 +54,10 @@ protected ElasticsearchContainer createSinkService(PulsarCluster cluster) { .withEnv("plugins.security.disabled", "true"); } + protected boolean isOpenSearch() { + return true; + } + @Override public void prepareSink() throws Exception { RestClientBuilder builder = RestClient.builder( diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index bd11b7d387383..ca5fe4b38527b 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -167,7 +167,9 @@ private PulsarCluster(PulsarClusterSpec spec, CSContainer csContainer, boolean s .withEnv("journalSyncData", "false") .withEnv("journalMaxGroupWaitMSec", "0") .withEnv("clusterName", clusterName) + .withEnv("PULSAR_PREFIX_diskUsageWarnThreshold", "0.95") .withEnv("diskUsageThreshold", "0.99") + .withEnv("PULSAR_PREFIX_diskUsageLwmThreshold", "0.97") .withEnv("nettyMaxFrameSizeBytes", String.valueOf(spec.maxMessageSize)); if (spec.bookkeeperEnvs != null) { bookieContainer.withEnv(spec.bookkeeperEnvs); From 2459928453203512e043a43cadb887bf17b6236e Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 30 Jun 2023 14:43:32 +0800 Subject: [PATCH 168/494] [fix] [admin] set ns level backlog quota does not take effect if retention exists (#20690) Motivation: When a retention policy exists, the command `pulsar-admin namespaces set-backlog-quota` does not take effect.In the PR https://github.com/apache/pulsar/pull/17383: it only checked the compatibility of `retention` and `backlog quota`, and it was not set. Modifications: Fix the bug. (cherry picked from commit 6acd01d70d9cf55cd4ae295c39cabdddc618c11f) --- .../broker/admin/impl/NamespacesBase.java | 1 + .../pulsar/broker/admin/AdminApi2Test.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 97029eb5ce128..aa338887abc29 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -1347,6 +1347,7 @@ protected CompletableFuture setBacklogQuotaAsync(BacklogQuotaType backlogQ "Backlog Quota exceeds configured retention quota for namespace." + " Please increase retention quota and retry"); } + policies.backlog_quota_map.put(quotaType, quota); return policies; }); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 71892d009313e..7d7d167fe3fb7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -3304,4 +3304,26 @@ private void testDeleteNamespaceForciblyWithManyTopics() throws Exception { admin.namespaces().deleteNamespace(ns, true); Assert.assertFalse(admin.namespaces().getNamespaces(defaultTenant).contains(ns)); } + + @Test + private void testSetBacklogQuotasNamespaceLevelIfRetentionExists() throws Exception { + final String ns = defaultTenant + "/ns-testSetBacklogQuotasNamespaceLevel"; + final long backlogQuotaLimitSize = 100000002; + final int backlogQuotaLimitTime = 2; + admin.namespaces().createNamespace(ns, 2); + // create retention. + admin.namespaces().setRetention(ns, new RetentionPolicies(1800, 10000)); + // set backlog quota. + admin.namespaces().setBacklogQuota(ns, BacklogQuota.builder() + .limitSize(backlogQuotaLimitSize).limitTime(backlogQuotaLimitTime).build()); + // Verify result. + Map map = admin.namespaces().getBacklogQuotaMap(ns); + assertEquals(map.size(), 1); + assertTrue(map.containsKey(BacklogQuota.BacklogQuotaType.destination_storage)); + BacklogQuota backlogQuota = map.get(BacklogQuota.BacklogQuotaType.destination_storage); + assertEquals(backlogQuota.getLimitSize(), backlogQuotaLimitSize); + assertEquals(backlogQuota.getLimitTime(), backlogQuotaLimitTime); + // cleanup. + admin.namespaces().deleteNamespace(ns); + } } From 1280225a402075bfd27c94dc316cc05c2ae54eb8 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 29 Jun 2023 10:54:31 +0800 Subject: [PATCH 169/494] [fix][broker] Fix the publish latency spike from the contention of MessageDeduplication (#20647) (cherry picked from commit 31c5b4dacce52f9f3698050060bafb5192fb6787) --- .../service/persistent/MessageDeduplication.java | 10 +++++----- .../service/persistent/MessageDuplicationTest.java | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index 030d74a014f29..ed4e70bfd2953 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -20,12 +20,12 @@ import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; @@ -126,7 +126,7 @@ public MessageDupUnknownException() { private final int maxNumberOfProducers; // Map used to track the inactive producer along with the timestamp of their last activity - private final Map inactiveProducers = new HashMap<>(); + private final Map inactiveProducers = new ConcurrentHashMap<>(); private final String replicatorPrefix; @@ -436,7 +436,7 @@ private boolean isDeduplicationEnabled() { /** * Topic will call this method whenever a producer connects. */ - public synchronized void producerAdded(String producerName) { + public void producerAdded(String producerName) { // Producer is no-longer inactive inactiveProducers.remove(producerName); } @@ -444,7 +444,7 @@ public synchronized void producerAdded(String producerName) { /** * Topic will call this method whenever a producer disconnects. */ - public synchronized void producerRemoved(String producerName) { + public void producerRemoved(String producerName) { // Producer is no-longer active inactiveProducers.put(producerName, System.currentTimeMillis()); } @@ -456,7 +456,7 @@ public synchronized void purgeInactiveProducers() { long minimumActiveTimestamp = System.currentTimeMillis() - TimeUnit.MINUTES .toMillis(pulsar.getConfiguration().getBrokerDeduplicationProducerInactivityTimeoutMinutes()); - Iterator> mapIterator = inactiveProducers.entrySet().iterator(); + Iterator> mapIterator = inactiveProducers.entrySet().iterator(); boolean hasInactive = false; while (mapIterator.hasNext()) { java.util.Map.Entry entry = mapIterator.next(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java index b4152b143ab6c..19583a4455ead 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java @@ -36,6 +36,7 @@ import io.netty.channel.EventLoopGroup; import java.lang.reflect.Field; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedger; @@ -174,7 +175,7 @@ public void testInactiveProducerRemove() throws Exception { Field field = MessageDeduplication.class.getDeclaredField("inactiveProducers"); field.setAccessible(true); - Map inactiveProducers = (Map) field.get(messageDeduplication); + Map inactiveProducers = (ConcurrentHashMap) field.get(messageDeduplication); String producerName1 = "test1"; when(publishContext.getHighestSequenceId()).thenReturn(2L); From 9c76b0902cdf4535f7cfbc4e8ee5833dc58c766d Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 29 Jun 2023 10:05:31 +0800 Subject: [PATCH 170/494] [fix][client]Fix deadlock issue of consumer while using multiple IO threads (#20669) (cherry picked from commit d3a8c2289925edcabfa01b5bb69f716e084c7360) --- .../client/api/MultiTopicsConsumerTest.java | 20 ++++++++++ .../client/impl/MultiTopicsConsumerImpl.java | 37 +++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java index b8ea87ab4016e..315ce378d6953 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java @@ -76,6 +76,11 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @Override + protected void customizeNewPulsarClientBuilder(ClientBuilder clientBuilder) { + clientBuilder.ioThreads(4).connectionsPerBroker(4); + } + // test that reproduces the issue https://github.com/apache/pulsar/issues/12024 // where closing the consumer leads to an endless receive loop @Test @@ -351,4 +356,19 @@ public int choosePartition(Message msg, TopicMetadata metadata) { } consumer.close(); } + + @Test(invocationCount = 10, timeOut = 30000) + public void testMultipleIOThreads() throws PulsarAdminException, PulsarClientException { + final var topic = TopicName.get(newTopicName()).toString(); + final var numPartitions = 100; + admin.topics().createPartitionedTopic(topic, numPartitions); + for (int i = 0; i < 100; i++) { + admin.topics().createNonPartitionedTopic(topic + "-" + i); + } + @Cleanup + final var consumer = pulsarClient.newConsumer(Schema.INT32).topicsPattern(topic + ".*") + .subscriptionName("sub").subscribe(); + assertTrue(consumer instanceof MultiTopicsConsumerImpl); + assertTrue(consumer.isConnected()); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index ef0345de919ea..a0cbcc18e65f1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -92,7 +92,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { // sum of topicPartitions, simple topic has 1, partitioned topic equals to partition number. AtomicInteger allTopicPartitionsNumber; - private boolean paused = false; + private volatile boolean paused = false; private final Object pauseMutex = new Object(); // timeout related to auto check and subscribe partition increasement private volatile Timeout partitionsAutoUpdateTimeout = null; @@ -1059,29 +1059,28 @@ private void doSubscribeTopicPartitions(Schema schema, CompletableFuture> subFuture = new CompletableFuture<>(); - consumers.compute(topicName, (key, existingValue) -> { - if (existingValue != null) { - String errorMessage = String.format("[%s] Failed to subscribe for topic [%s] in topics consumer. " - + "Topic is already being subscribed for in other thread.", topic, topicName); - log.warn(errorMessage); - subscribeResult.completeExceptionally(new PulsarClientException(errorMessage)); - return existingValue; - } else { - internalConfig.setStartPaused(paused); - ConsumerImpl newConsumer = createInternalConsumer(internalConfig, topicName, - -1, subFuture, createIfDoesNotExist, schema); - - synchronized (pauseMutex) { + synchronized (pauseMutex) { + consumers.compute(topicName, (key, existingValue) -> { + if (existingValue != null) { + String errorMessage = + String.format("[%s] Failed to subscribe for topic [%s] in topics consumer. " + + "Topic is already being subscribed for in other thread.", topic, topicName); + log.warn(errorMessage); + subscribeResult.completeExceptionally(new PulsarClientException(errorMessage)); + return existingValue; + } else { + internalConfig.setStartPaused(paused); + ConsumerImpl newConsumer = createInternalConsumer(internalConfig, topicName, + -1, subFuture, createIfDoesNotExist, schema); if (paused) { newConsumer.pause(); } else { newConsumer.resume(); } + return newConsumer; } - return newConsumer; - } - }); - + }); + } futureList = Collections.singletonList(subFuture); } @@ -1408,7 +1407,7 @@ private CompletableFuture subscribeIncreasedTopicPartitions(String topicNa } if (log.isDebugEnabled()) { log.debug("[{}] create consumer {} for partitionName: {}", - topicName, newConsumer.getTopic(), partitionName); + topicName, newConsumer.getTopic(), partitionName); } return subFuture; }) From ce530110cb91c266eb489d8ee3849ce9f361d849 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 30 Jun 2023 12:03:37 -0500 Subject: [PATCH 171/494] [fix][fn] Exit JVM when main thread throws exception (#20689) Fixes: https://github.com/apache/pulsar/issues/20688 ### Motivation When a function throws an exception that ends processing, we should exit the JVM. ### Modifications * Update `JavaInstanceMain` so that an exception leads to exiting the JVM. Since the class does not use any dependencies (see the class's Javadoc), we use reflection to shutdown logging. ### Verifying this change This change is a trivial rework / code cleanup without any test coverage. ### Documentation - [x] `doc-not-needed` (cherry picked from commit d83c7a6cc16dd374a976d7d535b5fcd325b5a812) --- .../functions/instance/JavaInstanceMain.java | 58 +++++++++++++++---- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/pulsar-functions/runtime-all/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceMain.java b/pulsar-functions/runtime-all/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceMain.java index 6fb7ff2606b6d..cdc6705b50aa9 100644 --- a/pulsar-functions/runtime-all/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceMain.java +++ b/pulsar-functions/runtime-all/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceMain.java @@ -54,6 +54,20 @@ public class JavaInstanceMain { private static final String FUNCTIONS_INSTANCE_CLASSPATH = "pulsar.functions.instance.classpath"; + private static final Method log4j2ShutdownMethod; + + static { + // use reflection to find org.apache.logging.log4j.LogManager.shutdown method + Method shutdownMethod = null; + try { + shutdownMethod = Class.forName("org.apache.logging.log4j.LogManager") + .getMethod("shutdown"); + } catch (ClassNotFoundException | NoSuchMethodException e) { + // ignore + } + log4j2ShutdownMethod = shutdownMethod; + } + public JavaInstanceMain() { } @@ -91,17 +105,27 @@ public static void main(String[] args) throws Exception { System.out.println("Using function root classloader: " + root); System.out.println("Using function instance classloader: " + functionInstanceClsLoader); - - // use the function instance classloader to create org.apache.pulsar.functions.runtime.JavaInstanceStarter - Object main = - createInstance("org.apache.pulsar.functions.runtime.JavaInstanceStarter", functionInstanceClsLoader); - - // Invoke start method of JavaInstanceStarter to start the function instance code - Method method = - main.getClass().getDeclaredMethod("start", String[].class, ClassLoader.class, ClassLoader.class); - - System.out.println("Starting function instance..."); - method.invoke(main, args, functionInstanceClsLoader, root); + try { + // use the function instance classloader to create org.apache.pulsar.functions.runtime.JavaInstanceStarter + Object main = + createInstance("org.apache.pulsar.functions.runtime.JavaInstanceStarter", + functionInstanceClsLoader); + + // Invoke start method of JavaInstanceStarter to start the function instance code + Method method = + main.getClass().getDeclaredMethod("start", String[].class, ClassLoader.class, ClassLoader.class); + + System.out.println("Starting function instance..."); + method.invoke(main, args, functionInstanceClsLoader, root); + } catch (Throwable e) { + try { + shutdownLogging(); + } finally { + System.out.println("Failed to start function instance."); + e.printStackTrace(); + Runtime.getRuntime().halt(1); + } + } } public static Object createInstance(String userClassName, @@ -152,4 +176,16 @@ public static boolean isBlank(String str) { return true; } } + + private static void shutdownLogging() { + // flush log buffers and shutdown log4j2 logging to prevent log truncation + if (log4j2ShutdownMethod != null) { + try { + // use reflection to call org.apache.logging.log4j.LogManager.shutdown() + log4j2ShutdownMethod.invoke(null); + } catch (IllegalAccessException | InvocationTargetException e) { + // ignore + } + } + } } From d115040c6167d4046c41f9ae057fa5ee78f8fd31 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Tue, 20 Jun 2023 10:51:50 +0800 Subject: [PATCH 172/494] [fix][broker]fix the publish latency spike issue with large number of producers (#20607) (cherry picked from commit 084d2abf1bd829d147959e2cf9aaf82d7cc73ac5) --- .../pulsar/broker/service/AbstractTopic.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 1371019be41dc..827069045509b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -131,7 +132,11 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener RATE_LIMITED_UPDATER = AtomicLongFieldUpdater.newUpdater(AbstractTopic.class, "publishRateLimitedTimes"); - protected volatile long publishRateLimitedTimes = 0; + protected volatile long publishRateLimitedTimes = 0L; + + private static final AtomicIntegerFieldUpdater USER_CREATED_PRODUCER_COUNTER_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(AbstractTopic.class, "userCreatedProducerCount"); + private volatile int userCreatedProducerCount = 0; protected volatile Optional topicEpoch = Optional.empty(); private volatile boolean hasExclusiveProducer; @@ -447,14 +452,8 @@ protected boolean isProducersExceeded(Producer producer) { return false; } Integer maxProducers = topicPolicies.getMaxProducersPerTopic().get(); - if (maxProducers != null && maxProducers > 0 && maxProducers <= getUserCreatedProducersSize()) { - return true; - } - return false; - } - - private long getUserCreatedProducersSize() { - return producers.values().stream().filter(p -> !p.isRemote()).count(); + return maxProducers != null && maxProducers > 0 + && maxProducers <= USER_CREATED_PRODUCER_COUNTER_UPDATER.get(this); } protected void registerTopicPolicyListener() { @@ -988,6 +987,8 @@ protected void internalAddProducer(Producer producer) throws BrokerServiceExcept Producer existProducer = producers.putIfAbsent(producer.getProducerName(), producer); if (existProducer != null) { tryOverwriteOldProducer(existProducer, producer); + } else if (!producer.isRemote()) { + USER_CREATED_PRODUCER_COUNTER_UPDATER.incrementAndGet(this); } } @@ -1020,6 +1021,9 @@ public void removeProducer(Producer producer) { checkArgument(producer.getTopic() == this); if (producers.remove(producer.getProducerName(), producer)) { + if (!producer.isRemote()) { + USER_CREATED_PRODUCER_COUNTER_UPDATER.decrementAndGet(this); + } handleProducerRemoved(producer); } } From 4a46a6f034f8196aa0e4b68cce01c47c8f336220 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Mon, 3 Jul 2023 14:22:40 +0800 Subject: [PATCH 173/494] [improve][broker] Upgrade bookkeeper to 4.16.2 (#20704) (cherry picked from commit 00c37450eb1dbfd973372555fec8292c44c07846) --- .../server/src/assemble/LICENSE.bin.txt | 56 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 6 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 30 +++++----- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b3a573ebc269b..4b8534a165035 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -346,34 +346,34 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-proto-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-server-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.1.jar - - org.apache.bookkeeper-circe-checksum-4.16.1.jar - - org.apache.bookkeeper-cpu-affinity-4.16.1.jar - - org.apache.bookkeeper-statelib-4.16.1.jar - - org.apache.bookkeeper-stream-storage-api-4.16.1.jar - - org.apache.bookkeeper-stream-storage-common-4.16.1.jar - - org.apache.bookkeeper-stream-storage-java-client-4.16.1.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.16.1.jar - - org.apache.bookkeeper-stream-storage-proto-4.16.1.jar - - org.apache.bookkeeper-stream-storage-server-4.16.1.jar - - org.apache.bookkeeper-stream-storage-service-api-4.16.1.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.16.1.jar - - org.apache.bookkeeper.http-http-server-4.16.1.jar - - org.apache.bookkeeper.http-vertx-http-server-4.16.1.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.1.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.1.jar - - org.apache.distributedlog-distributedlog-common-4.16.1.jar - - org.apache.distributedlog-distributedlog-core-4.16.1-tests.jar - - org.apache.distributedlog-distributedlog-core-4.16.1.jar - - org.apache.distributedlog-distributedlog-protocol-4.16.1.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.1.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.1.jar - - org.apache.bookkeeper-native-io-4.16.1.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.2.jar + - org.apache.bookkeeper-circe-checksum-4.16.2.jar + - org.apache.bookkeeper-cpu-affinity-4.16.2.jar + - org.apache.bookkeeper-statelib-4.16.2.jar + - org.apache.bookkeeper-stream-storage-api-4.16.2.jar + - org.apache.bookkeeper-stream-storage-common-4.16.2.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.2.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.2.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.2.jar + - org.apache.bookkeeper-stream-storage-server-4.16.2.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.2.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.2.jar + - org.apache.bookkeeper.http-http-server-4.16.2.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.2.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.2.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.2.jar + - org.apache.distributedlog-distributedlog-common-4.16.2.jar + - org.apache.distributedlog-distributedlog-core-4.16.2-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.2.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.2.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.2.jar + - org.apache.bookkeeper-native-io-4.16.2.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 3040091789cc0..d562a21bf7645 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -390,9 +390,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.16.1.jar - - cpu-affinity-4.16.1.jar - - circe-checksum-4.16.1.jar + - bookkeeper-common-allocator-4.16.2.jar + - cpu-affinity-4.16.2.jar + - circe-checksum-4.16.2.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index 6806efce101c7..9617efcdc6dc4 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.16.1 + 4.16.2 3.8.1 1.5.0 1.10.0 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 7f78a9c06f8a5..0623c0a306f17 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -426,21 +426,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.16.1.jar - - bookkeeper-common-allocator-4.16.1.jar - - bookkeeper-proto-4.16.1.jar - - bookkeeper-server-4.16.1.jar - - bookkeeper-stats-api-4.16.1.jar - - bookkeeper-tools-framework-4.16.1.jar - - circe-checksum-4.16.1.jar - - codahale-metrics-provider-4.16.1.jar - - cpu-affinity-4.16.1.jar - - http-server-4.16.1.jar - - prometheus-metrics-provider-4.16.1.jar - - codahale-metrics-provider-4.16.1.jar - - bookkeeper-slogger-api-4.16.1.jar - - bookkeeper-slogger-slf4j-4.16.1.jar - - native-io-4.16.1.jar + - bookkeeper-common-4.16.2.jar + - bookkeeper-common-allocator-4.16.2.jar + - bookkeeper-proto-4.16.2.jar + - bookkeeper-server-4.16.2.jar + - bookkeeper-stats-api-4.16.2.jar + - bookkeeper-tools-framework-4.16.2.jar + - circe-checksum-4.16.2.jar + - codahale-metrics-provider-4.16.2.jar + - cpu-affinity-4.16.2.jar + - http-server-4.16.2.jar + - prometheus-metrics-provider-4.16.2.jar + - codahale-metrics-provider-4.16.2.jar + - bookkeeper-slogger-api-4.16.2.jar + - bookkeeper-slogger-slf4j-4.16.2.jar + - native-io-4.16.2.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar From f8373dc4494072c8bbb9b2aff49bb1754bdf3ab1 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 29 Jun 2023 13:14:55 +0800 Subject: [PATCH 174/494] [fix][meta] Adding the missed bookie id in the registration manager. (#20641) --- .../pulsar/metadata/bookkeeper/PulsarRegistrationManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationManager.java index 32ec89e717e0e..25c3f10aa18fa 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationManager.java @@ -135,7 +135,7 @@ public void registerBookie(BookieId bookieId, boolean readOnly, BookieServiceInf if (readOnly) { ResourceLock rwRegistration = bookieRegistration.remove(bookieId); if (rwRegistration != null) { - log.info("Bookie {} was already registered as writable, unregistering"); + log.info("Bookie {} was already registered as writable, unregistering", bookieId); rwRegistration.release().get(); } @@ -144,7 +144,7 @@ public void registerBookie(BookieId bookieId, boolean readOnly, BookieServiceInf } else { ResourceLock roRegistration = bookieRegistrationReadOnly.remove(bookieId); if (roRegistration != null) { - log.info("Bookie {} was already registered as read-only, unregistering"); + log.info("Bookie {} was already registered as read-only, unregistering", bookieId); roRegistration.release().get(); } From 6da3a3644cc6639447360e3c1e39cc2ff1493d29 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Fri, 30 Jun 2023 09:21:15 +0800 Subject: [PATCH 175/494] [fix][meta] Bookie Info lost by notification race condition. (#20642) --- .../bookkeeper/PulsarRegistrationClient.java | 221 +++--- .../FaultInjectableZKRegistrationManager.java | 630 ++++++++++++++++++ .../PulsarRegistrationClientTest.java | 126 +++- 3 files changed, 874 insertions(+), 103 deletions(-) create mode 100644 pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/FaultInjectableZKRegistrationManager.java diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java index a32625926e72c..306b6398b5c50 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java @@ -18,13 +18,18 @@ */ package org.apache.pulsar.metadata.bookkeeper; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static java.util.concurrent.CompletableFuture.failedFuture; import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; import static org.apache.bookkeeper.util.BookKeeperConstants.COOKIE_NODE; import static org.apache.bookkeeper.util.BookKeeperConstants.READONLY; +import static org.apache.pulsar.common.util.FutureUtil.Sequencer; +import static org.apache.pulsar.common.util.FutureUtil.waitForAll; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -42,10 +47,10 @@ import org.apache.bookkeeper.versioning.Version; import org.apache.bookkeeper.versioning.Versioned; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.CacheGetResult; import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.Notification; -import org.apache.pulsar.metadata.api.NotificationType; @Slf4j public class PulsarRegistrationClient implements RegistrationClient { @@ -56,20 +61,22 @@ public class PulsarRegistrationClient implements RegistrationClient { private final String bookieRegistrationPath; private final String bookieAllRegistrationPath; private final String bookieReadonlyRegistrationPath; - - private final ConcurrentHashMap> bookieServiceInfoCache = - new ConcurrentHashMap(); private final Set writableBookiesWatchers = new CopyOnWriteArraySet<>(); private final Set readOnlyBookiesWatchers = new CopyOnWriteArraySet<>(); private final MetadataCache bookieServiceInfoMetadataCache; private final ScheduledExecutorService executor; + private final Map> writableBookieInfo; + private final Map> readOnlyBookieInfo; + private final FutureUtil.Sequencer sequencer; public PulsarRegistrationClient(MetadataStore store, String ledgersRootPath) { this.store = store; this.ledgersRootPath = ledgersRootPath; this.bookieServiceInfoMetadataCache = store.getMetadataCache(BookieServiceInfoSerde.INSTANCE); - + this.sequencer = Sequencer.create(); + this.writableBookieInfo = new ConcurrentHashMap<>(); + this.readOnlyBookieInfo = new ConcurrentHashMap<>(); // Following Bookie Network Address Changes is an expensive operation // as it requires additional ZooKeeper watches // we can disable this feature, in case the BK cluster has only @@ -77,7 +84,6 @@ public PulsarRegistrationClient(MetadataStore store, this.bookieRegistrationPath = ledgersRootPath + "/" + AVAILABLE_NODE; this.bookieAllRegistrationPath = ledgersRootPath + "/" + COOKIE_NODE; this.bookieReadonlyRegistrationPath = this.bookieRegistrationPath + "/" + READONLY; - this.executor = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-registration-client")); @@ -91,38 +97,62 @@ public void close() { @Override public CompletableFuture>> getWritableBookies() { - return getChildren(bookieRegistrationPath); + return getBookiesThenFreshCache(bookieRegistrationPath); } @Override public CompletableFuture>> getAllBookies() { // this method is meant to return all the known bookies, even the bookies // that are not in a running state - return getChildren(bookieAllRegistrationPath); + return getBookiesThenFreshCache(bookieAllRegistrationPath); } @Override public CompletableFuture>> getReadOnlyBookies() { - return getChildren(bookieReadonlyRegistrationPath); + return getBookiesThenFreshCache(bookieReadonlyRegistrationPath); } - private CompletableFuture>> getChildren(String path) { + /** + * @throws IllegalArgumentException if parameter path is null or empty. + */ + private CompletableFuture>> getBookiesThenFreshCache(String path) { + if (path == null || path.isEmpty()) { + return failedFuture( + new IllegalArgumentException("parameter [path] can not be null or empty.")); + } return store.getChildren(path) .thenComposeAsync(children -> { - Set bookieIds = PulsarRegistrationClient.convertToBookieAddresses(children); - List> bookieInfoUpdated = - new ArrayList<>(bookieIds.size()); + final Set bookieIds = PulsarRegistrationClient.convertToBookieAddresses(children); + final List> bookieInfoUpdated = new ArrayList<>(bookieIds.size()); for (BookieId id : bookieIds) { // update the cache for new bookies - if (!bookieServiceInfoCache.containsKey(id)) { - bookieInfoUpdated.add(readBookieServiceInfoAsync(id)); + if (path.equals(bookieReadonlyRegistrationPath) && readOnlyBookieInfo.get(id) == null) { + bookieInfoUpdated.add(readBookieInfoAsReadonlyBookie(id)); + continue; + } + if (path.equals(bookieRegistrationPath) && writableBookieInfo.get(id) == null) { + bookieInfoUpdated.add(readBookieInfoAsWritableBookie(id)); + continue; + } + if (path.equals(bookieAllRegistrationPath)) { + if (writableBookieInfo.get(id) != null || readOnlyBookieInfo.get(id) != null) { + // jump to next bookie id + continue; + } + // check writable first + final CompletableFuture revalidateAllBookiesFuture = readBookieInfoAsWritableBookie(id) + .thenCompose(writableBookieInfo -> writableBookieInfo + .>>>map( + bookieServiceInfo -> completedFuture(null)) + // check read-only then + .orElseGet(() -> readBookieInfoAsReadonlyBookie(id))); + bookieInfoUpdated.add(revalidateAllBookiesFuture); } } if (bookieInfoUpdated.isEmpty()) { - return CompletableFuture.completedFuture(bookieIds); + return completedFuture(bookieIds); } else { - return FutureUtil - .waitForAll(bookieInfoUpdated) + return waitForAll(bookieInfoUpdated) .thenApply(___ -> bookieIds); } }) @@ -153,42 +183,67 @@ public void unwatchReadOnlyBookies(RegistrationListener registrationListener) { readOnlyBookiesWatchers.remove(registrationListener); } - private void handleDeletedBookieNode(Notification n) { - if (n.getType() == NotificationType.Deleted) { - BookieId bookieId = stripBookieIdFromPath(n.getPath()); - if (bookieId != null) { - log.info("Bookie {} disappeared", bookieId); - bookieServiceInfoCache.remove(bookieId); - } + /** + * This method will receive metadata store notifications and then update the + * local cache in background sequentially. + */ + private void updatedBookies(Notification n) { + // make the notification callback run sequential in background. + final String path = n.getPath(); + if (!path.startsWith(bookieReadonlyRegistrationPath) && !path.startsWith(bookieRegistrationPath)) { + // ignore unknown path + return; } - } - - private void handleUpdatedBookieNode(Notification n) { - BookieId bookieId = stripBookieIdFromPath(n.getPath()); - if (bookieId != null) { - log.info("Bookie {} info updated", bookieId); - readBookieServiceInfoAsync(bookieId); + if (path.equals(bookieReadonlyRegistrationPath) || path.equals(bookieRegistrationPath)) { + // ignore root path + return; } - } - - private void updatedBookies(Notification n) { - if (n.getType() == NotificationType.Created || n.getType() == NotificationType.Deleted) { - if (n.getPath().startsWith(bookieReadonlyRegistrationPath)) { - getReadOnlyBookies().thenAccept(bookies -> { - readOnlyBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies))); - }); - handleDeletedBookieNode(n); - } else if (n.getPath().startsWith(bookieRegistrationPath)) { - getWritableBookies().thenAccept(bookies -> - writableBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); - handleDeletedBookieNode(n); - } - } else if (n.getType() == NotificationType.Modified) { - if (n.getPath().startsWith(bookieReadonlyRegistrationPath) - || n.getPath().startsWith(bookieRegistrationPath)) { - handleUpdatedBookieNode(n); + final BookieId bookieId = stripBookieIdFromPath(n.getPath()); + sequencer.sequential(() -> { + switch (n.getType()) { + case Created: + log.info("Bookie {} created. path: {}", bookieId, n.getPath()); + if (path.startsWith(bookieReadonlyRegistrationPath)) { + return getReadOnlyBookies().thenAccept(bookies -> + readOnlyBookiesWatchers.forEach(w -> + executor.execute(() -> w.onBookiesChanged(bookies)))); + } + return getWritableBookies().thenAccept(bookies -> + writableBookiesWatchers.forEach(w -> + executor.execute(() -> w.onBookiesChanged(bookies)))); + case Modified: + if (bookieId == null) { + return completedFuture(null); + } + log.info("Bookie {} modified. path: {}", bookieId, n.getPath()); + if (path.startsWith(bookieReadonlyRegistrationPath)) { + return readBookieInfoAsReadonlyBookie(bookieId).thenApply(__ -> null); + } + return readBookieInfoAsWritableBookie(bookieId).thenApply(__ -> null); + case Deleted: + if (bookieId == null) { + return completedFuture(null); + } + log.info("Bookie {} deleted. path: {}", bookieId, n.getPath()); + if (path.startsWith(bookieReadonlyRegistrationPath)) { + readOnlyBookieInfo.remove(bookieId); + return getReadOnlyBookies().thenAccept(bookies -> { + readOnlyBookiesWatchers.forEach(w -> + executor.execute(() -> w.onBookiesChanged(bookies))); + }); + } + if (path.startsWith(bookieRegistrationPath)) { + writableBookieInfo.remove(bookieId); + return getWritableBookies().thenAccept(bookies -> { + writableBookiesWatchers.forEach(w -> + executor.execute(() -> w.onBookiesChanged(bookies))); + }); + } + return completedFuture(null); + default: + return completedFuture(null); } - } + }); } private static BookieId stripBookieIdFromPath(String path) { @@ -200,7 +255,7 @@ private static BookieId stripBookieIdFromPath(String path) { try { return BookieId.parse(path.substring(slash + 1)); } catch (IllegalArgumentException e) { - log.warn("Cannot decode bookieId from {}", path, e); + log.warn("Cannot decode bookieId from {}, error: {}", path, e.getMessage()); } } return null; @@ -227,46 +282,48 @@ public CompletableFuture> getBookieServiceInfo(Book // this is because there are a few cases in which some operations on the main thread // wait for the result. This is due to the fact that resolving the address of a bookie // is needed in many code paths. - Versioned resultFromCache = bookieServiceInfoCache.get(bookieId); + Versioned info; + if ((info = writableBookieInfo.get(bookieId)) == null) { + info = readOnlyBookieInfo.get(bookieId); + } if (log.isDebugEnabled()) { - log.debug("getBookieServiceInfo {} -> {}", bookieId, resultFromCache); + log.debug("getBookieServiceInfo {} -> {}", bookieId, info); } - if (resultFromCache != null) { - return CompletableFuture.completedFuture(resultFromCache); + if (info != null) { + return completedFuture(info); } else { return FutureUtils.exception(new BKException.BKBookieHandleNotAvailableException()); } } - public CompletableFuture readBookieServiceInfoAsync(BookieId bookieId) { - String asWritable = bookieRegistrationPath + "/" + bookieId; - return bookieServiceInfoMetadataCache.get(asWritable) - .thenCompose((Optional getResult) -> { - if (getResult.isPresent()) { - Versioned res = - new Versioned<>(getResult.get(), new LongVersion(-1)); - log.info("Update BookieInfoCache (writable bookie) {} -> {}", bookieId, getResult.get()); - bookieServiceInfoCache.put(bookieId, res); - return CompletableFuture.completedFuture(null); - } else { - return readBookieInfoAsReadonlyBookie(bookieId); - } - } - ); + public CompletableFuture>> readBookieInfoAsWritableBookie( + BookieId bookieId) { + final String asWritable = bookieRegistrationPath + "/" + bookieId; + return bookieServiceInfoMetadataCache.getWithStats(asWritable) + .thenApply((Optional> bkInfoWithStats) -> { + if (bkInfoWithStats.isPresent()) { + final CacheGetResult r = bkInfoWithStats.get(); + log.info("Update BookieInfoCache (writable bookie) {} -> {}", bookieId, r.getValue()); + writableBookieInfo.put(bookieId, + new Versioned<>(r.getValue(), new LongVersion(r.getStat().getVersion()))); + } + return bkInfoWithStats; + } + ); } - final CompletableFuture readBookieInfoAsReadonlyBookie(BookieId bookieId) { - String asReadonly = bookieReadonlyRegistrationPath + "/" + bookieId; - return bookieServiceInfoMetadataCache.get(asReadonly) - .thenApply((Optional getResultAsReadOnly) -> { - if (getResultAsReadOnly.isPresent()) { - Versioned res = - new Versioned<>(getResultAsReadOnly.get(), new LongVersion(-1)); - log.info("Update BookieInfoCache (readonly bookie) {} -> {}", bookieId, - getResultAsReadOnly.get()); - bookieServiceInfoCache.put(bookieId, res); + final CompletableFuture>> readBookieInfoAsReadonlyBookie( + BookieId bookieId) { + final String asReadonly = bookieReadonlyRegistrationPath + "/" + bookieId; + return bookieServiceInfoMetadataCache.getWithStats(asReadonly) + .thenApply((Optional> bkInfoWithStats) -> { + if (bkInfoWithStats.isPresent()) { + final CacheGetResult r = bkInfoWithStats.get(); + log.info("Update BookieInfoCache (readonly bookie) {} -> {}", bookieId, r.getValue()); + readOnlyBookieInfo.put(bookieId, + new Versioned<>(r.getValue(), new LongVersion(r.getStat().getVersion()))); } - return null; + return bkInfoWithStats; }); } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/FaultInjectableZKRegistrationManager.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/FaultInjectableZKRegistrationManager.java new file mode 100644 index 0000000000000..bcbf41addbae3 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/FaultInjectableZKRegistrationManager.java @@ -0,0 +1,630 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.metadata.bookkeeper; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; +import static org.apache.bookkeeper.util.BookKeeperConstants.COOKIE_NODE; +import static org.apache.bookkeeper.util.BookKeeperConstants.EMPTY_BYTE_ARRAY; +import static org.apache.bookkeeper.util.BookKeeperConstants.INSTANCEID; +import static org.apache.bookkeeper.util.BookKeeperConstants.READONLY; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.bookie.BookieException.BookieIllegalOpException; +import org.apache.bookkeeper.bookie.BookieException.CookieExistException; +import org.apache.bookkeeper.bookie.BookieException.CookieNotFoundException; +import org.apache.bookkeeper.bookie.BookieException.MetadataStoreException; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BKException.BKInterruptedException; +import org.apache.bookkeeper.client.BKException.MetaStoreException; +import org.apache.bookkeeper.common.concurrent.FutureUtils; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.discover.BookieServiceInfo; +import org.apache.bookkeeper.discover.RegistrationClient; +import org.apache.bookkeeper.discover.RegistrationManager; +import org.apache.bookkeeper.discover.ZKRegistrationClient; +import org.apache.bookkeeper.meta.AbstractZkLedgerManagerFactory; +import org.apache.bookkeeper.meta.LayoutManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.ZkLayoutManager; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.DataFormats.BookieServiceInfoFormat; +import org.apache.bookkeeper.util.BookKeeperConstants; +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.bookkeeper.versioning.LongVersion; +import org.apache.bookkeeper.versioning.Version; +import org.apache.bookkeeper.versioning.Versioned; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NoNodeException; +import org.apache.zookeeper.KeeperException.NodeExistsException; +import org.apache.zookeeper.Op; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.Watcher.Event.KeeperState; +import org.apache.zookeeper.ZKUtil; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; + +/** + * Fault injectable ZK registration manager. + * Copy from #{@link org.apache.bookkeeper.discover.ZKRegistrationManager}. + */ +@Slf4j +public class FaultInjectableZKRegistrationManager implements RegistrationManager { + + private static final Function EXCEPTION_FUNC = cause -> { + if (cause instanceof BKException) { + log.error("Failed to get bookie list : ", cause); + return (BKException) cause; + } else if (cause instanceof InterruptedException) { + log.error("Interrupted reading bookie list : ", cause); + return new BKInterruptedException(); + } else { + return new MetaStoreException(); + } + }; + + private final ServerConfiguration conf; + private final ZooKeeper zk; + private final List zkAcls; + private final LayoutManager layoutManager; + + private volatile boolean zkRegManagerInitialized = false; + + // ledgers root path + private final String ledgersRootPath; + // cookie path + private final String cookiePath; + // registration paths + protected final String bookieRegistrationPath; + protected final String bookieReadonlyRegistrationPath; + // session timeout in milliseconds + private final int zkTimeoutMs; + private final List listeners = new ArrayList<>(); + private Function hookOnRegisterReadOnly; + + public FaultInjectableZKRegistrationManager(ServerConfiguration conf, + ZooKeeper zk) { + this(conf, zk, ZKMetadataDriverBase.resolveZkLedgersRootPath(conf)); + } + + public FaultInjectableZKRegistrationManager(ServerConfiguration conf, + ZooKeeper zk, + String ledgersRootPath) { + this.conf = conf; + this.zk = zk; + this.zkAcls = ZkUtils.getACLs(conf); + this.ledgersRootPath = ledgersRootPath; + this.cookiePath = ledgersRootPath + "/" + COOKIE_NODE; + this.bookieRegistrationPath = ledgersRootPath + "/" + AVAILABLE_NODE; + this.bookieReadonlyRegistrationPath = this.bookieRegistrationPath + "/" + READONLY; + this.zkTimeoutMs = conf.getZkTimeout(); + + this.layoutManager = new ZkLayoutManager( + zk, + ledgersRootPath, + zkAcls); + + this.zk.register(event -> { + if (!zkRegManagerInitialized) { + // do nothing until first registration + return; + } + // Check for expired connection. + if (event.getType().equals(EventType.None) + && event.getState().equals(KeeperState.Expired)) { + listeners.forEach(RegistrationListener::onRegistrationExpired); + } + }); + } + + @Override + public void close() { + // no-op + } + + /** + * Returns the CookiePath of the bookie in the ZooKeeper. + * + * @param bookieId bookie id + * @return + */ + public String getCookiePath(BookieId bookieId) { + return this.cookiePath + "/" + bookieId; + } + + // + // Registration Management + // + + /** + * Check existence of regPath and wait it expired if possible. + * + * @param regPath reg node path. + * @return true if regPath exists, otherwise return false + * @throws IOException if can't create reg path + */ + protected boolean checkRegNodeAndWaitExpired(String regPath) throws IOException { + final CountDownLatch prevNodeLatch = new CountDownLatch(1); + Watcher zkPrevRegNodewatcher = new Watcher() { + @Override + public void process(WatchedEvent event) { + // Check for prev znode deletion. Connection expiration is + // not handling, since bookie has logic to shutdown. + if (EventType.NodeDeleted == event.getType()) { + prevNodeLatch.countDown(); + } + } + }; + try { + Stat stat = zk.exists(regPath, zkPrevRegNodewatcher); + if (null != stat) { + // if the ephemeral owner isn't current zookeeper client + // wait for it to be expired. + if (stat.getEphemeralOwner() != zk.getSessionId()) { + log.info("Previous bookie registration znode: {} exists, so waiting zk sessiontimeout:" + + " {} ms for znode deletion", regPath, zkTimeoutMs); + // waiting for the previous bookie reg znode deletion + if (!prevNodeLatch.await(zkTimeoutMs, TimeUnit.MILLISECONDS)) { + throw new NodeExistsException(regPath); + } else { + return false; + } + } + return true; + } else { + return false; + } + } catch (KeeperException ke) { + log.error("ZK exception checking and wait ephemeral znode {} expired : ", regPath, ke); + throw new IOException("ZK exception checking and wait ephemeral znode " + + regPath + " expired", ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + log.error("Interrupted checking and wait ephemeral znode {} expired : ", regPath, ie); + throw new IOException("Interrupted checking and wait ephemeral znode " + + regPath + " expired", ie); + } + } + + @Override + public void registerBookie(BookieId bookieId, boolean readOnly, + BookieServiceInfo bookieServiceInfo) throws BookieException { + if (!readOnly) { + String regPath = bookieRegistrationPath + "/" + bookieId; + doRegisterBookie(regPath, bookieServiceInfo); + } else { + doRegisterReadOnlyBookie(bookieId, bookieServiceInfo); + } + } + + @VisibleForTesting + static byte[] serializeBookieServiceInfo(BookieServiceInfo bookieServiceInfo) { + if (log.isDebugEnabled()) { + log.debug("serialize BookieServiceInfo {}", bookieServiceInfo); + } + try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { + BookieServiceInfoFormat.Builder builder = BookieServiceInfoFormat.newBuilder(); + List bsiEndpoints = bookieServiceInfo.getEndpoints().stream() + .map(e -> { + return BookieServiceInfoFormat.Endpoint.newBuilder() + .setId(e.getId()) + .setPort(e.getPort()) + .setHost(e.getHost()) + .setProtocol(e.getProtocol()) + .addAllAuth(e.getAuth()) + .addAllExtensions(e.getExtensions()) + .build(); + }) + .collect(Collectors.toList()); + + builder.addAllEndpoints(bsiEndpoints); + builder.putAllProperties(bookieServiceInfo.getProperties()); + + builder.build().writeTo(os); + return os.toByteArray(); + } catch (IOException err) { + log.error("Cannot serialize bookieServiceInfo from " + bookieServiceInfo); + throw new RuntimeException(err); + } + } + + private void doRegisterBookie(String regPath, BookieServiceInfo bookieServiceInfo) throws BookieException { + // ZK ephemeral node for this Bookie. + try { + if (!checkRegNodeAndWaitExpired(regPath)) { + // Create the ZK ephemeral node for this Bookie. + zk.create(regPath, serializeBookieServiceInfo(bookieServiceInfo), zkAcls, CreateMode.EPHEMERAL); + zkRegManagerInitialized = true; + } + } catch (KeeperException ke) { + log.error("ZK exception registering ephemeral Znode for Bookie!", ke); + // Throw an IOException back up. This will cause the Bookie + // constructor to error out. Alternatively, we could do a System + // exit here as this is a fatal error. + throw new MetadataStoreException(ke); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + log.error("Interrupted exception registering ephemeral Znode for Bookie!", ie); + // Throw an IOException back up. This will cause the Bookie + // constructor to error out. Alternatively, we could do a System + // exit here as this is a fatal error. + throw new MetadataStoreException(ie); + } catch (IOException e) { + throw new MetadataStoreException(e); + } + } + + private void doRegisterReadOnlyBookie(BookieId bookieId, BookieServiceInfo bookieServiceInfo) + throws BookieException { + try { + if (null == zk.exists(this.bookieReadonlyRegistrationPath, false)) { + try { + zk.create(this.bookieReadonlyRegistrationPath, serializeBookieServiceInfo(bookieServiceInfo), + zkAcls, CreateMode.PERSISTENT); + } catch (NodeExistsException e) { + // this node is just now created by someone. + } + } + String regPath = bookieReadonlyRegistrationPath + "/" + bookieId; + doRegisterBookie(regPath, bookieServiceInfo); + // clear the write state + regPath = bookieRegistrationPath + "/" + bookieId; + try { + if (hookOnRegisterReadOnly != null) { + hookOnRegisterReadOnly.apply(null); + } + // Clear the current registered node + zk.delete(regPath, -1); + } catch (KeeperException.NoNodeException nne) { + log.warn("No writable bookie registered node {} when transitioning to readonly", + regPath, nne); + } + } catch (KeeperException | InterruptedException e) { + throw new MetadataStoreException(e); + } + } + + @Override + public void unregisterBookie(BookieId bookieId, boolean readOnly) throws BookieException { + String regPath; + if (!readOnly) { + regPath = bookieRegistrationPath + "/" + bookieId; + } else { + regPath = bookieReadonlyRegistrationPath + "/" + bookieId; + } + doUnregisterBookie(regPath); + } + + private void doUnregisterBookie(String regPath) throws BookieException { + try { + zk.delete(regPath, -1); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new MetadataStoreException(ie); + } catch (KeeperException e) { + throw new MetadataStoreException(e); + } + } + + // + // Cookie Management + // + + @Override + public void writeCookie(BookieId bookieId, + Versioned cookieData) throws BookieException { + String zkPath = getCookiePath(bookieId); + try { + if (Version.NEW == cookieData.getVersion()) { + if (zk.exists(cookiePath, false) == null) { + try { + zk.create(cookiePath, new byte[0], zkAcls, CreateMode.PERSISTENT); + } catch (NodeExistsException nne) { + log.info("More than one bookie tried to create {} at once. Safe to ignore.", + cookiePath); + } + } + zk.create(zkPath, cookieData.getValue(), zkAcls, CreateMode.PERSISTENT); + } else { + if (!(cookieData.getVersion() instanceof LongVersion)) { + throw new BookieIllegalOpException("Invalid version type, expected it to be LongVersion"); + } + zk.setData( + zkPath, + cookieData.getValue(), + (int) ((LongVersion) cookieData.getVersion()).getLongVersion()); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new MetadataStoreException("Interrupted writing cookie for bookie " + bookieId, ie); + } catch (NoNodeException nne) { + throw new CookieNotFoundException(bookieId.toString()); + } catch (NodeExistsException nee) { + throw new CookieExistException(bookieId.toString()); + } catch (KeeperException e) { + throw new MetadataStoreException("Failed to write cookie for bookie " + bookieId); + } + } + + @Override + public Versioned readCookie(BookieId bookieId) throws BookieException { + String zkPath = getCookiePath(bookieId); + try { + Stat stat = zk.exists(zkPath, false); + byte[] data = zk.getData(zkPath, false, stat); + // sets stat version from ZooKeeper + LongVersion version = new LongVersion(stat.getVersion()); + return new Versioned<>(data, version); + } catch (NoNodeException nne) { + throw new CookieNotFoundException(bookieId.toString()); + } catch (KeeperException | InterruptedException e) { + throw new MetadataStoreException("Failed to read cookie for bookie " + bookieId); + } + } + + @Override + public void removeCookie(BookieId bookieId, Version version) throws BookieException { + String zkPath = getCookiePath(bookieId); + try { + zk.delete(zkPath, (int) ((LongVersion) version).getLongVersion()); + } catch (NoNodeException e) { + throw new CookieNotFoundException(bookieId.toString()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new MetadataStoreException("Interrupted deleting cookie for bookie " + bookieId, e); + } catch (KeeperException e) { + throw new MetadataStoreException("Failed to delete cookie for bookie " + bookieId); + } + + log.info("Removed cookie from {} for bookie {}.", cookiePath, bookieId); + } + + + @Override + public String getClusterInstanceId() throws BookieException { + String instanceId = null; + try { + if (zk.exists(ledgersRootPath, null) == null) { + log.error("BookKeeper metadata doesn't exist in zookeeper. " + + "Has the cluster been initialized? " + + "Try running bin/bookkeeper shell metaformat"); + throw new KeeperException.NoNodeException("BookKeeper metadata"); + } + try { + byte[] data = zk.getData(ledgersRootPath + "/" + + INSTANCEID, false, null); + instanceId = new String(data, UTF_8); + } catch (KeeperException.NoNodeException e) { + log.info("INSTANCEID not exists in zookeeper. Not considering it for data verification"); + } + } catch (KeeperException | InterruptedException e) { + throw new MetadataStoreException("Failed to get cluster instance id", e); + } + return instanceId; + } + + @Override + public boolean prepareFormat() throws Exception { + boolean ledgerRootExists = null != zk.exists(ledgersRootPath, false); + boolean availableNodeExists = null != zk.exists(bookieRegistrationPath, false); + // Create ledgers root node if not exists + if (!ledgerRootExists) { + ZkUtils.createFullPathOptimistic(zk, ledgersRootPath, "".getBytes(StandardCharsets.UTF_8), zkAcls, + CreateMode.PERSISTENT); + } + // create available bookies node if not exists + if (!availableNodeExists) { + zk.create(bookieRegistrationPath, "".getBytes(StandardCharsets.UTF_8), zkAcls, CreateMode.PERSISTENT); + } + + // create readonly bookies node if not exists + if (null == zk.exists(bookieReadonlyRegistrationPath, false)) { + zk.create(bookieReadonlyRegistrationPath, new byte[0], zkAcls, CreateMode.PERSISTENT); + } + + return ledgerRootExists; + } + + @Override + public boolean initNewCluster() throws Exception { + String zkServers = ZKMetadataDriverBase.resolveZkServers(conf); + String instanceIdPath = ledgersRootPath + "/" + INSTANCEID; + log.info("Initializing ZooKeeper metadata for new cluster, ZKServers: {} ledger root path: {}", zkServers, + ledgersRootPath); + + boolean ledgerRootExists = null != zk.exists(ledgersRootPath, false); + + if (ledgerRootExists) { + log.error("Ledger root path: {} already exists", ledgersRootPath); + return false; + } + + List multiOps = Lists.newArrayListWithExpectedSize(4); + + // Create ledgers root node + multiOps.add(Op.create(ledgersRootPath, EMPTY_BYTE_ARRAY, zkAcls, CreateMode.PERSISTENT)); + + // create available bookies node + multiOps.add(Op.create(bookieRegistrationPath, EMPTY_BYTE_ARRAY, zkAcls, CreateMode.PERSISTENT)); + + // create readonly bookies node + multiOps.add(Op.create( + bookieReadonlyRegistrationPath, + EMPTY_BYTE_ARRAY, + zkAcls, + CreateMode.PERSISTENT)); + + // create INSTANCEID + String instanceId = UUID.randomUUID().toString(); + multiOps.add(Op.create(instanceIdPath, instanceId.getBytes(UTF_8), + zkAcls, CreateMode.PERSISTENT)); + + // execute the multi ops + zk.multi(multiOps); + + // creates the new layout and stores in zookeeper + AbstractZkLedgerManagerFactory.newLedgerManagerFactory(conf, layoutManager); + + log.info("Successfully initiated cluster. ZKServers: {} ledger root path: {} instanceId: {}", zkServers, + ledgersRootPath, instanceId); + return true; + } + + @Override + public boolean nukeExistingCluster() throws Exception { + String zkServers = ZKMetadataDriverBase.resolveZkServers(conf); + log.info("Nuking ZooKeeper metadata of existing cluster, ZKServers: {} ledger root path: {}", + zkServers, ledgersRootPath); + + boolean ledgerRootExists = null != zk.exists(ledgersRootPath, false); + if (!ledgerRootExists) { + log.info("There is no existing cluster with ledgersRootPath: {} in ZKServers: {}, " + + "so exiting nuke operation", ledgersRootPath, zkServers); + return true; + } + + boolean availableNodeExists = null != zk.exists(bookieRegistrationPath, false); + try (RegistrationClient regClient = new ZKRegistrationClient( + zk, + ledgersRootPath, + null, + false + )) { + if (availableNodeExists) { + Collection rwBookies = FutureUtils + .result(regClient.getWritableBookies(), EXCEPTION_FUNC).getValue(); + if (rwBookies != null && !rwBookies.isEmpty()) { + log.error("Bookies are still up and connected to this cluster, " + + "stop all bookies before nuking the cluster"); + return false; + } + + boolean readonlyNodeExists = null != zk.exists(bookieReadonlyRegistrationPath, false); + if (readonlyNodeExists) { + Collection roBookies = FutureUtils + .result(regClient.getReadOnlyBookies(), EXCEPTION_FUNC).getValue(); + if (roBookies != null && !roBookies.isEmpty()) { + log.error("Readonly Bookies are still up and connected to this cluster, " + + "stop all bookies before nuking the cluster"); + return false; + } + } + } + } + + LedgerManagerFactory ledgerManagerFactory = + AbstractZkLedgerManagerFactory.newLedgerManagerFactory(conf, layoutManager); + return ledgerManagerFactory.validateAndNukeExistingCluster(conf, layoutManager); + } + + @Override + public boolean format() throws Exception { + // Clear underreplicated ledgers + try { + ZKUtil.deleteRecursive(zk, ZkLedgerUnderreplicationManager.getBasePath(ledgersRootPath) + + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH); + } catch (KeeperException.NoNodeException e) { + if (log.isDebugEnabled()) { + log.debug("underreplicated ledgers root path node not exists in zookeeper to delete"); + } + } + + // Clear underreplicatedledger locks + try { + ZKUtil.deleteRecursive(zk, ZkLedgerUnderreplicationManager.getBasePath(ledgersRootPath) + '/' + + BookKeeperConstants.UNDER_REPLICATION_LOCK); + } catch (KeeperException.NoNodeException e) { + if (log.isDebugEnabled()) { + log.debug("underreplicatedledger locks node not exists in zookeeper to delete"); + } + } + + // Clear the cookies + try { + ZKUtil.deleteRecursive(zk, cookiePath); + } catch (KeeperException.NoNodeException e) { + if (log.isDebugEnabled()) { + log.debug("cookies node not exists in zookeeper to delete"); + } + } + + // Clear the INSTANCEID + try { + zk.delete(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID, -1); + } catch (KeeperException.NoNodeException e) { + if (log.isDebugEnabled()) { + log.debug("INSTANCEID not exists in zookeeper to delete"); + } + } + + // create INSTANCEID + String instanceId = UUID.randomUUID().toString(); + zk.create(ledgersRootPath + "/" + BookKeeperConstants.INSTANCEID, + instanceId.getBytes(StandardCharsets.UTF_8), zkAcls, CreateMode.PERSISTENT); + + log.info("Successfully formatted BookKeeper metadata"); + return true; + } + + @Override + public boolean isBookieRegistered(BookieId bookieId) throws BookieException { + String regPath = bookieRegistrationPath + "/" + bookieId; + String readonlyRegPath = bookieReadonlyRegistrationPath + "/" + bookieId; + try { + return ((null != zk.exists(regPath, false)) || (null != zk.exists(readonlyRegPath, false))); + } catch (KeeperException e) { + log.error("ZK exception while checking registration ephemeral znodes for BookieId: {}", bookieId, e); + throw new MetadataStoreException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("InterruptedException while checking registration ephemeral znodes for BookieId: {}", bookieId, + e); + throw new MetadataStoreException(e); + } + } + + @Override + public void addRegistrationListener(RegistrationListener listener) { + listeners.add(listener); + } + + public void betweenRegisterReadOnlyBookie(Function fn) { + hookOnRegisterReadOnly = fn; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java index f599451c00710..4dcbcda3d9078 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java @@ -42,9 +42,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.conf.AbstractConfiguration; -import org.apache.bookkeeper.discover.BookieServiceInfo; -import org.apache.bookkeeper.discover.RegistrationClient; -import org.apache.bookkeeper.discover.RegistrationManager; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.discover.*; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.versioning.Version; @@ -52,6 +51,7 @@ import org.apache.pulsar.metadata.BaseMetadataStoreTest; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.zookeeper.ZooKeeper; import org.awaitility.Awaitility; import org.testng.annotations.Test; @@ -126,7 +126,7 @@ public void testGetReadonlyBookies(String provider, Supplier urlSupplier public void testGetBookieServiceInfo(String provider, Supplier urlSupplier) throws Exception { @Cleanup MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), - MetadataStoreConfig.builder().fsyncEnable(false).build()); + MetadataStoreConfig.builder().fsyncEnable(false).build()); String ledgersRoot = "/test/ledgers-" + UUID.randomUUID(); @@ -168,10 +168,10 @@ public void testGetBookieServiceInfo(String provider, Supplier urlSuppli getAndVerifyAllBookies(rc, addresses); Awaitility.await().untilAsserted(() -> { - for (BookieId address : addresses) { - BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); - compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(address)); - }}); + for (BookieId address : addresses) { + BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); + compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(address)); + }}); // shutdown the bookies (but keep the cookie) for (BookieId address : addresses) { @@ -184,12 +184,12 @@ public void testGetBookieServiceInfo(String provider, Supplier urlSuppli // getBookieServiceInfo should fail with BKBookieHandleNotAvailableException Awaitility.await().untilAsserted(() -> { - for (BookieId address : addresses) { - assertTrue( - expectThrows(ExecutionException.class, () -> { - rc.getBookieServiceInfo(address).get(); - }).getCause() instanceof BKException.BKBookieHandleNotAvailableException); - }}); + for (BookieId address : addresses) { + assertTrue( + expectThrows(ExecutionException.class, () -> { + rc.getBookieServiceInfo(address).get(); + }).getCause() instanceof BKException.BKBookieHandleNotAvailableException); + }}); // restart the bookies, all writable @@ -241,12 +241,12 @@ public void testGetBookieServiceInfo(String provider, Supplier urlSuppli .await() .ignoreExceptionsMatching(e -> e.getCause() instanceof BKException.BKBookieHandleNotAvailableException) .untilAsserted(() -> { - // verify that infos are updated - for (BookieId address : addresses) { - BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); - compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(address)); - } - }); + // verify that infos are updated + for (BookieId address : addresses) { + BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); + compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(address)); + } + }); } @@ -318,7 +318,7 @@ private void testWatchBookiesSuccess(String provider, Supplier urlSuppli @Cleanup MetadataStoreExtended store = MetadataStoreExtended.create(urlSupplier.get(), - MetadataStoreConfig.builder().fsyncEnable(false).build()); + MetadataStoreConfig.builder().fsyncEnable(false).build()); String ledgersRoot = "/test/ledgers-" + UUID.randomUUID(); @@ -357,4 +357,88 @@ private void testWatchBookiesSuccess(String provider, Supplier urlSuppli }); } + + @Test + public void testNetworkDelayWithBkZkManager() throws Throwable { + final String zksConnectionString = zks.getConnectionString(); + final String ledgersRoot = "/test/ledgers-" + UUID.randomUUID(); + // prepare registration manager + ZooKeeper zk = new ZooKeeper(zksConnectionString, 5000, null); + final ServerConfiguration serverConfiguration = new ServerConfiguration(); + serverConfiguration.setZkLedgersRootPath(ledgersRoot); + final FaultInjectableZKRegistrationManager rm = new FaultInjectableZKRegistrationManager(serverConfiguration, zk); + rm.prepareFormat(); + // prepare registration client + @Cleanup + MetadataStoreExtended store = MetadataStoreExtended.create(zksConnectionString, + MetadataStoreConfig.builder().fsyncEnable(false).build()); + @Cleanup + RegistrationClient rc1 = new PulsarRegistrationClient(store, ledgersRoot); + @Cleanup + RegistrationClient rc2 = new PulsarRegistrationClient(store, ledgersRoot); + + final List addresses = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + addresses.add(BookieId.parse("BOOKIE-" + i)); + } + final Map bookieServiceInfos = new HashMap<>(); + + int port = 223; + for (BookieId address : addresses) { + BookieServiceInfo info = new BookieServiceInfo(); + BookieServiceInfo.Endpoint endpoint = new BookieServiceInfo.Endpoint(); + endpoint.setAuth(Collections.emptyList()); + endpoint.setExtensions(Collections.emptyList()); + endpoint.setId("id"); + endpoint.setHost("localhost"); + endpoint.setPort(port++); + endpoint.setProtocol("bookie-rpc"); + info.setEndpoints(Arrays.asList(endpoint)); + bookieServiceInfos.put(address, info); + rm.registerBookie(address, false, info); + // write the cookie + rm.writeCookie(address, new Versioned<>(new byte[0], Version.NEW)); + } + + // trigger loading the BookieServiceInfo in the local cache + getAndVerifyAllBookies(rc1, addresses); + getAndVerifyAllBookies(rc2, addresses); + + Awaitility.await().untilAsserted(() -> { + for (BookieId address : addresses) { + compareBookieServiceInfo(rc1.getBookieServiceInfo(address).get().getValue(), + bookieServiceInfos.get(address)); + compareBookieServiceInfo(rc2.getBookieServiceInfo(address).get().getValue(), + bookieServiceInfos.get(address)); + } + }); + + // verified the init status. + + + // mock network delay + rm.betweenRegisterReadOnlyBookie(__ -> { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return null; + }); + + for (int i = 0; i < addresses.size() / 2; i++) { + final BookieId bkId = addresses.get(i); + // turn some bookies to be read only. + rm.registerBookie(bkId, true, bookieServiceInfos.get(bkId)); + } + + Awaitility.await().untilAsserted(() -> { + for (BookieId address : addresses) { + compareBookieServiceInfo(rc1.getBookieServiceInfo(address).get().getValue(), + bookieServiceInfos.get(address)); + compareBookieServiceInfo(rc2.getBookieServiceInfo(address).get().getValue(), + bookieServiceInfos.get(address)); + } + }); + } } From aa565c21c8103be8edbafaffa90029f90bce37da Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Fri, 30 Jun 2023 14:01:24 +0800 Subject: [PATCH 176/494] [fix][broker] Fix NPE when reset Replicator's cursor by position. (#20597) --- .../admin/impl/PersistentTopicsBase.java | 42 ++++++++++++------- .../PersistentMessageExpiryMonitor.java | 16 ++++--- .../persistent/PersistentReplicator.java | 2 +- .../persistent/PersistentSubscription.java | 2 +- .../service/PersistentMessageFinderTest.java | 20 ++++++--- .../pulsar/broker/service/ReplicatorTest.java | 30 +++++++++++++ 6 files changed, 84 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 81f5e3c1f323b..03e1a1b27475a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -4145,31 +4145,43 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit return; } try { - PersistentSubscription sub = topic.getSubscription(subName); - if (sub == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - return; + PersistentSubscription sub = null; + PersistentReplicator repl = null; + + if (subName.startsWith(topic.getReplicatorPrefix())) { + String remoteCluster = PersistentReplicator.getRemoteCluster(subName); + repl = (PersistentReplicator) + topic.getPersistentReplicator(remoteCluster); + if (repl == null) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + "Replicator not found")); + return; + } + } else { + sub = topic.getSubscription(subName); + if (sub == null) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + getSubNotFoundErrorMessage(topicName.toString(), subName))); + return; + } } + CompletableFuture batchSizeFuture = new CompletableFuture<>(); getEntryBatchSize(batchSizeFuture, topic, messageId, batchIndex); + + PersistentReplicator finalRepl = repl; + PersistentSubscription finalSub = sub; + batchSizeFuture.thenAccept(bi -> { PositionImpl position = calculatePositionAckSet(isExcluded, bi, batchIndex, messageId); boolean issued; try { if (subName.startsWith(topic.getReplicatorPrefix())) { - String remoteCluster = PersistentReplicator.getRemoteCluster(subName); - PersistentReplicator repl = (PersistentReplicator) - topic.getPersistentReplicator(remoteCluster); - if (repl == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - "Replicator not found")); - return; - } - issued = repl.expireMessages(position); + issued = finalRepl.expireMessages(position); } else { - issued = sub.expireMessages(position); + issued = finalSub.expireMessages(position); } + if (issued) { log.info("[{}] Message expire started up to {} on {} {}", clientAppId(), position, topicName, subName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index bddcb1b334df1..3acd683f9f42b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -22,6 +22,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.LongAdder; +import javax.annotation.Nullable; import org.apache.bookkeeper.mledger.AsyncCallbacks.FindEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -43,6 +44,7 @@ public class PersistentMessageExpiryMonitor implements FindEntryCallback { private final ManagedCursor cursor; private final String subName; + private final PersistentTopic topic; private final String topicName; private final Rate msgExpired; private final LongAdder totalMsgExpired; @@ -57,9 +59,10 @@ public class PersistentMessageExpiryMonitor implements FindEntryCallback { expirationCheckInProgressUpdater = AtomicIntegerFieldUpdater .newUpdater(PersistentMessageExpiryMonitor.class, "expirationCheckInProgress"); - public PersistentMessageExpiryMonitor(String topicName, String subscriptionName, ManagedCursor cursor, - PersistentSubscription subscription) { - this.topicName = topicName; + public PersistentMessageExpiryMonitor(PersistentTopic topic, String subscriptionName, ManagedCursor cursor, + @Nullable PersistentSubscription subscription) { + this.topic = topic; + this.topicName = topic.getName(); this.cursor = cursor; this.subName = subscriptionName; this.subscription = subscription; @@ -98,11 +101,12 @@ public boolean expireMessages(int messageTTLInSeconds) { public boolean expireMessages(Position messagePosition) { // If it's beyond last position of this topic, do nothing. - if (((PositionImpl) subscription.getTopic().getLastPosition()).compareTo((PositionImpl) messagePosition) < 0) { + PositionImpl topicLastPosition = (PositionImpl) this.topic.getLastPosition(); + if (topicLastPosition.compareTo((PositionImpl) messagePosition) < 0) { if (log.isDebugEnabled()) { log.debug("[{}][{}] Ignore expire-message scheduled task, given position {} is beyond " - + "current topic's last position {}", topicName, subName, messagePosition, - subscription.getTopic().getLastPosition()); + + "current topic's last position {}", topicName, subName, messagePosition, + topicLastPosition); } return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index ccf70eecec35c..d468cb0643f72 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -116,7 +116,7 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man brokerService, replicationClient); this.topic = localTopic; this.cursor = cursor; - this.expiryMonitor = new PersistentMessageExpiryMonitor(localTopicName, + this.expiryMonitor = new PersistentMessageExpiryMonitor(localTopic, Codec.decode(cursor.getName()), cursor, null); HAVE_PENDING_READ_UPDATER.set(this, FALSE); PENDING_MESSAGES_UPDATER.set(this, 0); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 090e8e253776d..192e5ed3e006a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -152,7 +152,7 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.topicName = topic.getName(); this.subName = subscriptionName; this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString(); - this.expiryMonitor = new PersistentMessageExpiryMonitor(topicName, subscriptionName, cursor, this); + this.expiryMonitor = new PersistentMessageExpiryMonitor(topic, subscriptionName, cursor, this); this.setReplicated(replicated); this.subscriptionProperties = MapUtils.isEmpty(subscriptionProperties) ? Collections.emptyMap() : Collections.unmodifiableMap(subscriptionProperties); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java index 48798f0020f01..3fdf5d672996f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java @@ -62,6 +62,7 @@ import org.apache.pulsar.broker.service.persistent.PersistentMessageExpiryMonitor; import org.apache.pulsar.broker.service.persistent.PersistentMessageFinder; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ResetCursorData; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; @@ -230,7 +231,11 @@ public void findEntryFailed(ManagedLedgerException exception, Optional }); assertTrue(ex.get()); - PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor("topicname", c1.getName(), c1, null); + PersistentTopic mock = mock(PersistentTopic.class); + when(mock.getName()).thenReturn("topicname"); + when(mock.getLastPosition()).thenReturn(PositionImpl.EARLIEST); + + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); monitor.findEntryFailed(new ManagedLedgerException.ConcurrentFindCursorPositionException("failed"), Optional.empty(), null); Field field = monitor.getClass().getDeclaredField("expirationCheckInProgress"); @@ -374,7 +379,11 @@ void testMessageExpiryWithTimestampNonRecoverableException() throws Exception { bkc.deleteLedger(ledgers.get(1).getLedgerId()); bkc.deleteLedger(ledgers.get(2).getLedgerId()); - PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor("topicname", c1.getName(), c1, null); + PersistentTopic mock = mock(PersistentTopic.class); + when(mock.getName()).thenReturn("topicname"); + when(mock.getLastPosition()).thenReturn(PositionImpl.EARLIEST); + + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); Position previousMarkDelete = null; for (int i = 0; i < totalEntries; i++) { monitor.expireMessages(1); @@ -411,15 +420,16 @@ void testMessageExpiryWithPosition() throws Exception { ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); PersistentSubscription subscription = mock(PersistentSubscription.class); - Topic topic = mock(Topic.class); + PersistentTopic topic = mock(PersistentTopic.class); when(subscription.getTopic()).thenReturn(topic); + when(topic.getName()).thenReturn("topicname"); for (int i = 0; i < totalEntries; i++) { positions.add(ledger.addEntry(createMessageWrittenToLedger("msg" + i))); } when(topic.getLastPosition()).thenReturn(positions.get(positions.size() - 1)); - PersistentMessageExpiryMonitor monitor = spy(new PersistentMessageExpiryMonitor("topicname", + PersistentMessageExpiryMonitor monitor = spy(new PersistentMessageExpiryMonitor(topic, cursor.getName(), cursor, subscription)); assertEquals(cursor.getMarkDeletedPosition(), PositionImpl.get(positions.get(0).getLedgerId(), -1)); boolean issued; @@ -458,7 +468,7 @@ void testMessageExpiryWithPosition() throws Exception { clearInvocations(monitor); ManagedCursorImpl mockCursor = mock(ManagedCursorImpl.class); - PersistentMessageExpiryMonitor mockMonitor = spy(new PersistentMessageExpiryMonitor("topicname", + PersistentMessageExpiryMonitor mockMonitor = spy(new PersistentMessageExpiryMonitor(topic, cursor.getName(), mockCursor, subscription)); // Not calling findEntryComplete to clear expirationCheckInProgress condition, so following call to // expire message shouldn't issue. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index 176eab0e94b3d..f710c8541d1b5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -728,6 +728,36 @@ public void testReplicatorClearBacklog() throws Exception { assertEquals(status.getReplicationBacklog(), 0); } + + @Test(timeOut = 30000) + public void testResetReplicatorSubscriptionPosition() throws Exception { + final TopicName dest = TopicName + .get(BrokerTestUtil.newUniqueName("persistent://pulsar/ns/resetReplicatorSubscription")); + + @Cleanup + MessageProducer producer1 = new MessageProducer(url1, dest); + + // Produce from cluster1 and consume from the rest + for (int i = 0; i < 10; i++) { + producer1.produce(2); + } + + PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString()).get(); + + PersistentReplicator replicator = (PersistentReplicator) spy( + topic.getReplicators().get(topic.getReplicators().keys().get(0))); + + MessageId id = topic.getLastMessageId().get(); + admin1.topics().expireMessages(dest.getPartitionedTopicName(), + replicator.getCursor().getName(), + id,false); + + replicator.updateRates(); + + ReplicatorStats status = replicator.getStats(); + assertEquals(status.getReplicationBacklog(), 0); + } + @Test(timeOut = 30000) public void testResetCursorNotFail() throws Exception { From 8f5aa1e236b80b3e3ebdcca41c5bb051a0d58b8d Mon Sep 17 00:00:00 2001 From: Hao Zhang Date: Fri, 30 Jun 2023 12:28:32 +0800 Subject: [PATCH 177/494] [fix][broker] Topic policy can not be work well if replay policy message has any exception. (#20613) --- .../SystemTopicBasedTopicPoliciesService.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 9b10055f36fac..90c1705d3616f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -180,7 +180,11 @@ private void notifyListener(Message msg) { TopicName topicName = TopicName.get(TopicName.get(msg.getKey()).getPartitionedTopicName()); if (listeners.get(topicName) != null) { for (TopicPolicyListener listener : listeners.get(topicName)) { - listener.onUpdate(null); + try { + listener.onUpdate(null); + } catch (Throwable error) { + log.error("[{}] call listener error.", topicName, error); + } } } return; @@ -195,7 +199,11 @@ private void notifyListener(Message msg) { if (listeners.get(topicName) != null) { TopicPolicies policies = event.getPolicies(); for (TopicPolicyListener listener : listeners.get(topicName)) { - listener.onUpdate(policies); + try { + listener.onUpdate(policies); + } catch (Throwable error) { + log.error("[{}] call listener error.", topicName, error); + } } } } @@ -365,7 +373,11 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp policiesCache.forEach(((topicName, topicPolicies) -> { if (listeners.get(topicName) != null) { for (TopicPolicyListener listener : listeners.get(topicName)) { - listener.onUpdate(topicPolicies); + try { + listener.onUpdate(topicPolicies); + } catch (Throwable error) { + log.error("[{}] call listener error.", topicName, error); + } } } })); From df440aec9b902aa1a14d9dc83b0656f4ac3c3d26 Mon Sep 17 00:00:00 2001 From: hanmz Date: Thu, 29 Jun 2023 19:47:48 +0800 Subject: [PATCH 178/494] [fix][broker] Fix return the earliest position when query position by timestamp. (#20457) --- .../bookkeeper/mledger/ManagedCursor.java | 17 ++++++++++ .../mledger/impl/ManagedCursorImpl.java | 12 ++++++- .../impl/ManagedCursorContainerTest.java | 5 +++ .../persistent/PersistentMessageFinder.java | 2 +- .../broker/delayed/MockManagedCursor.java | 6 ++++ .../service/PersistentMessageFinderTest.java | 33 +++++++++++++++++++ 6 files changed, 73 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java index edbfa0b43204e..d1ffdf6d2d763 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java @@ -637,6 +637,23 @@ Position findNewestMatching(FindPositionConstraint constraint, Predicate void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, FindEntryCallback callback, Object ctx); + /** + * Find the newest entry that matches the given predicate. + * + * @param constraint + * search only active entries or all entries + * @param condition + * predicate that reads an entry an applies a condition + * @param callback + * callback object returning the resultant position + * @param ctx + * opaque context + * @param isFindFromLedger + * find the newest entry from ledger + */ + void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + FindEntryCallback callback, Object ctx, boolean isFindFromLedger); + /** * reset the cursor to specified position to enable replay of messages. * diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 1ce0403a54762..e5d784d40d48e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1182,6 +1182,12 @@ public void findEntryFailed(ManagedLedgerException exception, Optional @Override public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, FindEntryCallback callback, Object ctx) { + asyncFindNewestMatching(constraint, condition, callback, ctx, false); + } + + @Override + public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + FindEntryCallback callback, Object ctx, boolean isFindFromLedger) { OpFindNewest op; PositionImpl startPosition = null; long max = 0; @@ -1203,7 +1209,11 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate Optional.empty(), ctx); return; } - op = new OpFindNewest(this, startPosition, condition, max, callback, ctx); + if (isFindFromLedger) { + op = new OpFindNewest(this.ledger, startPosition, condition, max, callback, ctx); + } else { + op = new OpFindNewest(this, startPosition, condition, max, callback, ctx); + } op.find(); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java index 2c01b778caf6b..04d99d3bdf480 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java @@ -258,6 +258,11 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate AsyncCallbacks.FindEntryCallback callback, Object ctx) { } + @Override + public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + AsyncCallbacks.FindEntryCallback callback, Object ctx, boolean isFindFromLedger) { + } + @Override public void asyncResetCursor(final Position position, boolean forceReset, AsyncCallbacks.ResetCursorCallback callback) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageFinder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageFinder.java index d2e6f6f5ff869..08273155e4cfa 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageFinder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageFinder.java @@ -71,7 +71,7 @@ public void findMessages(final long timestamp, AsyncCallbacks.FindEntryCallback entry.release(); } return false; - }, this, callback); + }, this, callback, true); } else { if (log.isDebugEnabled()) { log.debug("[{}][{}] Ignore message position find scheduled task, last find is still running", topicName, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java index 499262c1e60b9..477290fc6837a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/delayed/MockManagedCursor.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Predicate; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.Entry; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -276,6 +277,11 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, } + @Override + public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition, + AsyncCallbacks.FindEntryCallback callback, Object ctx, boolean isFindFromLedger) { + } + @Override public void resetCursor(Position position) throws InterruptedException, ManagedLedgerException { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java index 3fdf5d672996f..f0e2e6eafcdfb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java @@ -248,6 +248,39 @@ public void findEntryFailed(ManagedLedgerException exception, Optional factory.shutdown(); } + @Test + void testPersistentMessageFinderWhenLastMessageDelete() throws Exception { + final String ledgerAndCursorName = "testPersistentMessageFinderWhenLastMessageDelete"; + + ManagedLedgerConfig config = new ManagedLedgerConfig(); + config.setRetentionSizeInMB(10); + config.setMaxEntriesPerLedger(10); + config.setRetentionTime(1, TimeUnit.HOURS); + ManagedLedger ledger = factory.open(ledgerAndCursorName, config); + ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); + + ledger.addEntry(createMessageWrittenToLedger("msg1")); + ledger.addEntry(createMessageWrittenToLedger("msg2")); + ledger.addEntry(createMessageWrittenToLedger("msg3")); + Position lastPosition = ledger.addEntry(createMessageWrittenToLedger("last-message")); + + long endTimestamp = System.currentTimeMillis() + 1000; + + Result result = new Result(); + // delete last position message + cursor.delete(lastPosition); + CompletableFuture future = findMessage(result, cursor, endTimestamp); + future.get(); + assertNull(result.exception); + assertNotEquals(result.position, null); + assertEquals(result.position, lastPosition); + + result.reset(); + cursor.close(); + ledger.close(); + factory.shutdown(); + } + @Test void testPersistentMessageFinderWithBrokerTimestampForMessage() throws Exception { From 46d057c51dac3558ba6478cd79728422154396d5 Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Tue, 13 Jun 2023 08:19:28 +0800 Subject: [PATCH 179/494] [fix] [Perf] PerformanceProducer do not produce expected number of messages. (#19775) Co-authored-by: tison --- .../pulsar/testclient/PerformanceProducer.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 0c56f1d5c736d..f6d8dc4f3d13c 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -480,6 +480,7 @@ public static void main(String[] args) throws Exception { oldTime = now; } + PerfClientUtils.exit(0); } private static void executorShutdownNow() { @@ -553,6 +554,7 @@ private static void runProducer(int producerId, byte[] payloadBytes, CountDownLatch doneLatch) { PulsarClient client = null; + boolean produceEnough = false; try { // Now processing command line arguments List>> futures = new ArrayList<>(); @@ -617,6 +619,9 @@ private static void runProducer(int producerId, AtomicLong numMessageSend = new AtomicLong(0); Semaphore numMsgPerTxnLimit = new Semaphore(arguments.numMessagesPerTransaction); while (true) { + if (produceEnough) { + break; + } for (Producer producer : producers) { if (arguments.testTime > 0) { if (System.nanoTime() > testEndTime) { @@ -624,7 +629,8 @@ private static void runProducer(int producerId, + "--------------", arguments.testTime); doneLatch.countDown(); Thread.sleep(5000); - PerfClientUtils.exit(0); + produceEnough = true; + break; } } @@ -634,7 +640,8 @@ private static void runProducer(int producerId, , numMessages); doneLatch.countDown(); Thread.sleep(5000); - PerfClientUtils.exit(0); + produceEnough = true; + break; } } rateLimiter.acquire(); @@ -763,10 +770,12 @@ private static void runProducer(int producerId, } catch (Throwable t) { log.error("Got error", t); } finally { + if (!produceEnough) { + doneLatch.countDown(); + } if (null != client) { try { client.close(); - PerfClientUtils.exit(1); } catch (PulsarClientException e) { log.error("Failed to close test client", e); } From ce519667cc311fd9b779eafa4f0ae6088f2dc266 Mon Sep 17 00:00:00 2001 From: YingQun Zhong Date: Fri, 16 Jun 2023 12:52:53 +0800 Subject: [PATCH 180/494] [fix][offload] Filesystem offloader class not found hadoop-hdfs-client (#20365) --- tiered-storage/file-system/pom.xml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index d3e8537e65500..5e03e5172b9d3 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -52,6 +52,34 @@ + + + org.apache.hadoop + hadoop-hdfs-client + ${hdfs-offload-version3} + + + org.apache.avro + avro + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + javax.servlet + servlet-api + + + org.apache.avro From 42894e70e26c894572efae00094fcf37b0769e33 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Fri, 16 Jun 2023 10:40:19 +0800 Subject: [PATCH 181/494] [fix][txn] Use PulsarResource check for topic existence instead of brokerservice.getTopic() (#20569) --- ...napshotSegmentAbortedTxnProcessorImpl.java | 11 ++-- .../buffer/TransactionBufferClientTest.java | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 4f4e58ac3f55b..fc92754d3008e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -58,6 +58,7 @@ import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; @@ -381,10 +382,12 @@ public void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Ob // This method will be deprecated and removed in version 4.x.0 private CompletableFuture recoverOldSnapshot() { - return topic.getBrokerService().getTopic(TopicName.get(topic.getName()).getNamespace() + "/" - + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, false) - .thenCompose(topicOption -> { - if (!topicOption.isPresent()) { + return topic.getBrokerService().getPulsar().getPulsarResources().getTopicResources() + .listPersistentTopicsAsync(NamespaceName.get(TopicName.get(topic.getName()).getNamespace())) + .thenCompose(topics -> { + if (!topics.contains(TopicDomain.persistent + "://" + + TopicName.get(topic.getName()).getNamespace() + "/" + + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT)) { return CompletableFuture.completedFuture(null); } else { return topic.getBrokerService().getPulsar().getTransactionBufferSnapshotServiceFactory() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index 86606e3ae8eb1..2cfc9f46f0ebe 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -42,15 +42,21 @@ import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferClientImpl; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.transaction.Transaction; import org.apache.pulsar.client.api.transaction.TransactionBufferClient; import org.apache.pulsar.client.api.transaction.TransactionBufferClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.api.proto.TxnAction; +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; @@ -115,6 +121,50 @@ public void testCommitOnTopic() throws ExecutionException, InterruptedException } } + @Test + public void testRecoveryTransactionBufferWhenCommonTopicAndSystemTopicAtDifferentBroker() throws Exception { + for (int i = 0; i < getPulsarServiceList().size(); i++) { + getPulsarServiceList().get(i).getConfig().setTransactionBufferSegmentedSnapshotEnabled(true); + } + String topic1 = NAMESPACE1 + "/testRecoveryTransactionBufferWhenCommonTopicAndSystemTopicAtDifferentBroker"; + admin.tenants().createTenant(TENANT, + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NAMESPACE1, 4); + admin.tenants().createTenant(NamespaceName.SYSTEM_NAMESPACE.getTenant(), + new TenantInfoImpl(Sets.newHashSet("appid1"), Sets.newHashSet(CLUSTER_NAME))); + admin.namespaces().createNamespace(NamespaceName.SYSTEM_NAMESPACE.toString()); + pulsarServiceList.get(0).getPulsarResources() + .getNamespaceResources() + .getPartitionedTopicResources() + .createPartitionedTopic(SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN, + new PartitionedTopicMetadata(3)); + assertTrue(admin.namespaces().getBundles(NAMESPACE1).getNumBundles() > 1); + for (int i = 0; true ; i++) { + topic1 = topic1 + i; + admin.topics().createNonPartitionedTopic(topic1); + String segmentTopicBroker = admin.lookups() + .lookupTopic(NAMESPACE1 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + String indexTopicBroker = admin.lookups() + .lookupTopic(NAMESPACE1 + "/" + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT_INDEXES); + if (segmentTopicBroker.equals(indexTopicBroker)) { + String topicBroker = admin.lookups().lookupTopic(topic1); + if (!topicBroker.equals(segmentTopicBroker)) { + break; + } + } else { + break; + } + } + pulsarClient = PulsarClient.builder().serviceUrl(pulsarServiceList.get(0).getBrokerServiceUrl()) + .enableTransaction(true).build(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.BYTES).topic(topic1).create(); + Transaction transaction = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build().get(); + producer.newMessage(transaction).send(); + } + @Test public void testAbortOnTopic() throws ExecutionException, InterruptedException { List> futures = new ArrayList<>(); From 119a8945c75f0351dcc4aadfd5a3866131e25c0d Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 1 Jul 2023 02:23:45 +0800 Subject: [PATCH 182/494] [fix][io] Close the kafka source connector got stuck (#20698) Motivation: https://github.com/apache/pulsar/discussions/19880#discussioncomment-6026807 When Kafka connector is closing, it waits for the `runnerThread` to stop, but the task-close is running at the same thread, so it will be stuck. Modifications: run `close` in another thread. --- pulsar-io/kafka/pom.xml | 6 ++++++ .../pulsar/io/kafka/KafkaAbstractSource.java | 14 ++++++++------ .../io/kafka/source/KafkaAbstractSourceTest.java | 6 +++++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 2d8fc93ed12ff..d6a6668f191c6 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -109,6 +109,12 @@ test + + org.awaitility + awaitility + test + + diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java index 8d2cbd8e74e14..012e4143744e8 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java @@ -190,12 +190,14 @@ public void start() { }); runnerThread.setUncaughtExceptionHandler( (t, e) -> { - LOG.error("[{}] Error while consuming records", t.getName(), e); - try { - this.close(); - } catch (InterruptedException ex) { - // The interrupted exception is thrown by the runnerThread itself. Ignore it. - } + new Thread(() -> { + LOG.error("[{}] Error while consuming records", t.getName(), e); + try { + this.close(); + } catch (Exception ex) { + LOG.error("[{}] Close kafka source error", t.getName(), e); + } + }, "Kafka Source Close Task Thread").start(); }); runnerThread.setName("Kafka Source Thread"); runnerThread.start(); diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java index bc06c3e1935b4..402727f4ec015 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java @@ -31,6 +31,7 @@ import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.kafka.KafkaAbstractSource; import org.apache.pulsar.io.kafka.KafkaSourceConfig; +import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -173,7 +174,10 @@ public final void closeConnectorWhenUnexpectedExceptionThrownTest() throws Excep Field runningField = KafkaAbstractSource.class.getDeclaredField("running"); runningField.setAccessible(true); - Assert.assertFalse((boolean) runningField.get(source)); + Awaitility.await().untilAsserted(() -> { + Assert.assertFalse((boolean) runningField.get(source)); + Assert.assertNull(consumerField.get(source)); + }); } private File getFile(String name) { From 4d8051ce08e4a6ced82253c6b7aed6aa1028b17f Mon Sep 17 00:00:00 2001 From: ran Date: Thu, 29 Jun 2023 17:01:58 +0800 Subject: [PATCH 183/494] [fix][sql] Remove useless configuration for Pulsar SQL (#20605) --- .../resources/conf/catalog/pulsar.properties | 4 --- .../sql/presto/TestPulsarConnectorConfig.java | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties b/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties index f15cd2657e0b5..3d6e367d5782b 100644 --- a/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties +++ b/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties @@ -113,10 +113,6 @@ pulsar.bookkeeper-explicit-interval=0 # running in same sql worker. 0 is represents disable the cache, default is 0. pulsar.managed-ledger-cache-size-MB = 0 -# Number of threads to be used for managed ledger tasks dispatching, -# default is Runtime.getRuntime().availableProcessors(). -# pulsar.managed-ledger-num-worker-threads = - # Number of threads to be used for managed ledger scheduled tasks, # default is Runtime.getRuntime().availableProcessors(). # pulsar.managed-ledger-num-scheduler-threads = diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java index 4f9dcf03979f7..b2b69fca90e5b 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarConnectorConfig.java @@ -18,7 +18,13 @@ */ package org.apache.pulsar.sql.presto; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.json.JsonModule; +import io.trino.spi.type.TypeManager; +import java.util.HashMap; +import java.util.Map; import org.apache.pulsar.common.policies.data.OffloadPoliciesImpl; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -100,4 +106,34 @@ public void testGetOffloadPolices() throws Exception { Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadServiceEndpoint(), endpoint); } + @Test + public void testAnnotatedConfigurations() { + Bootstrap app = new Bootstrap( + new JsonModule(), + new PulsarConnectorModule("connectorId", Mockito.mock(TypeManager.class))); + + Map config = new HashMap<>(); + + config.put("pulsar.managed-ledger-offload-driver", "aws-s3"); + config.put("pulsar.offloaders-directory", "/pulsar/offloaders"); + config.put("pulsar.managed-ledger-offload-max-threads", "2"); + config.put("pulsar.offloader-properties", "{\"s3ManagedLedgerOffloadBucket\":\"offload-bucket\"," + + "\"s3ManagedLedgerOffloadRegion\":\"us-west-2\"," + + "\"s3ManagedLedgerOffloadServiceEndpoint\":\"http://s3.amazonaws.com\"}"); + config.put("pulsar.auth-plugin", "org.apache.pulsar.client.impl.auth.AuthenticationToken"); + config.put("pulsar.auth-params", "params"); + config.put("pulsar.tls-allow-insecure-connection", "true"); + config.put("pulsar.tls-hostname-verification-enable", "true"); + config.put("pulsar.tls-trust-cert-file-path", "/path"); + config.put("pulsar.bookkeeper-num-io-threads", "10"); + config.put("pulsar.bookkeeper-num-worker-threads", "10"); + config.put("pulsar.managed-ledger-num-scheduler-threads", "10"); + config.put("pulsar.stats-provider", "org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider"); + config.put("pulsar.stats-provider-configs", "{\"httpServerEnabled\":\"false\"," + + "\"prometheusStatsHttpPort\":\"9092\"," + + "\"prometheusStatsHttpEnable\":\"true\"}"); + + app.doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize(); + } + } From dcc44d8fb251926ac080f1bbb7c85e046967bc81 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Fri, 30 Jun 2023 09:23:31 +0800 Subject: [PATCH 184/494] [improve][broker] Make get list from bundle Admin API async (#20652) --- .../broker/admin/v2/NonPersistentTopics.java | 59 +++++++++---------- .../pulsar/broker/web/PulsarWebResource.java | 16 ++--- .../ExtensibleLoadManagerImplTest.java | 13 ++-- 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java index cc269d02831d8..372bb0162527e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/NonPersistentTopics.java @@ -51,7 +51,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.web.RestException; -import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.EntryFilters; import org.apache.pulsar.common.policies.data.NamespaceOperation; @@ -445,35 +444,35 @@ public void getListFromBundle( bundleRange); asyncResponse.resume(Response.noContent().build()); } else { - NamespaceBundle nsBundle; - try { - nsBundle = validateNamespaceBundleOwnership(namespaceName, policies.bundles, - bundleRange, true, true); - } catch (WebApplicationException wae) { - asyncResponse.resume(wae); - return; - } - try { - ConcurrentOpenHashMap> bundleTopics = - pulsar().getBrokerService().getMultiLayerTopicsMap().get(namespaceName.toString()); - if (bundleTopics == null || bundleTopics.isEmpty()) { - asyncResponse.resume(Collections.emptyList()); - return; - } - final List topicList = new ArrayList<>(); - String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange(); - ConcurrentOpenHashMap topicMap = bundleTopics.get(bundleKey); - if (topicMap != null) { - topicList.addAll(topicMap.keys().stream() - .filter(name -> !TopicName.get(name).isPersistent()) - .collect(Collectors.toList())); - } - asyncResponse.resume(topicList); - } catch (Exception e) { - log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), - namespaceName, bundleRange, e); - asyncResponse.resume(new RestException(e)); - } + validateNamespaceBundleOwnershipAsync(namespaceName, policies.bundles, bundleRange, true, true) + .thenAccept(nsBundle -> { + ConcurrentOpenHashMap> bundleTopics = + pulsar().getBrokerService() + .getMultiLayerTopicsMap().get(namespaceName.toString()); + if (bundleTopics == null || bundleTopics.isEmpty()) { + asyncResponse.resume(Collections.emptyList()); + return; + } + final List topicList = new ArrayList<>(); + String bundleKey = namespaceName.toString() + "/" + nsBundle.getBundleRange(); + ConcurrentOpenHashMap topicMap = bundleTopics.get(bundleKey); + if (topicMap != null) { + topicList.addAll(topicMap.keys().stream() + .filter(name -> !TopicName.get(name).isPersistent()) + .collect(Collectors.toList())); + } + asyncResponse.resume(topicList); + }).exceptionally(ex -> { + Throwable realCause = FutureUtil.unwrapCompletionException(ex); + log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), + namespaceName, bundleRange, realCause); + if (realCause instanceof WebApplicationException) { + asyncResponse.resume(realCause); + } else { + asyncResponse.resume(new RestException(realCause)); + } + return null; + }); } }).exceptionally(ex -> { log.error("[{}] Failed to list topics on namespace bundle {}/{}", clientAppId(), diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index 3d23d7812543a..bfb94aa7740a6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -717,19 +717,19 @@ public CompletableFuture validateBundleOwnershipAsync(NamespaceBundle bund throw new RestException(Status.PRECONDITION_FAILED, "Failed to find ownership for ServiceUnit:" + bundle.toString()); } - // If the load manager is extensible load manager, we don't need check the authoritative. - if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { - return CompletableFuture.completedFuture(null); - } return nsService.isServiceUnitOwnedAsync(bundle) .thenAccept(owned -> { if (!owned) { boolean newAuthoritative = this.isLeaderBroker(); // Replace the host and port of the current request and redirect - URI redirect = UriBuilder.fromUri(uri.getRequestUri()).host(webUrl.get().getHost()) - .port(webUrl.get().getPort()).replaceQueryParam("authoritative", - newAuthoritative).replaceQueryParam("destinationBroker", - null).build(); + UriBuilder uriBuilder = UriBuilder.fromUri(uri.getRequestUri()) + .host(webUrl.get().getHost()) + .port(webUrl.get().getPort()) + .replaceQueryParam("authoritative", newAuthoritative); + if (!ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config())) { + uriBuilder.replaceQueryParam("destinationBroker", null); + } + URI redirect = uriBuilder.build(); log.debug("{} is not a service unit owned", bundle); // Redirect log.debug("Redirecting the rest call to {}", redirect); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 7af10c4aadcc5..0b51f27928bb7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -964,7 +964,10 @@ public void testDisableBroker() throws Exception { defaultConf.setAllowAutoTopicCreation(true); defaultConf.setForceDeleteNamespaceAllowed(true); defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); defaultConf.setLoadBalancerSheddingEnabled(false); + defaultConf.setLoadBalancerDebugModeEnabled(true); + defaultConf.setTopicLevelPoliciesEnabled(false); try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { var pulsar3 = additionalPulsarTestContext.getPulsarService(); ExtensibleLoadManagerImpl ternaryLoadManager = spy((ExtensibleLoadManagerImpl) @@ -1005,7 +1008,7 @@ public void testDisableBroker() throws Exception { @Test(timeOut = 30 * 1000) public void testListTopic() throws Exception { final String namespace = "public/testListTopic"; - admin.namespaces().createNamespace(namespace, 3); + admin.namespaces().createNamespace(namespace, 9); final String persistentTopicName = TopicName.get( "persistent", NamespaceName.get(namespace), @@ -1014,8 +1017,8 @@ public void testListTopic() throws Exception { final String nonPersistentTopicName = TopicName.get( "non-persistent", NamespaceName.get(namespace), "get_topics_mode_" + UUID.randomUUID()).toString(); - admin.topics().createPartitionedTopic(persistentTopicName, 3); - admin.topics().createPartitionedTopic(nonPersistentTopicName, 3); + admin.topics().createPartitionedTopic(persistentTopicName, 9); + admin.topics().createPartitionedTopic(nonPersistentTopicName, 9); pulsarClient.newProducer().topic(persistentTopicName).create().close(); pulsarClient.newProducer().topic(nonPersistentTopicName).create().close(); @@ -1033,10 +1036,10 @@ public void testListTopic() throws Exception { assertFalse(TopicName.get(s).isPersistent()); } } - assertEquals(topicNum, 3); + assertEquals(topicNum, 9); List list = admin.topics().getList(namespace); - assertEquals(list.size(), 6); + assertEquals(list.size(), 18); admin.namespaces().deleteNamespace(namespace, true); } From c595a51d6825f41bce38de24ccb2edca5e3ae6ad Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 30 Jun 2023 16:54:17 +0800 Subject: [PATCH 185/494] [fix][client] Make the whole grabCnx() progress atomic (#20595) --- .../client/impl/ConnectionHandlerTest.java | 154 ++++++++++++++++++ .../pulsar/client/impl/ConnectionHandler.java | 50 ++++-- .../pulsar/client/impl/ConsumerImpl.java | 26 +-- .../pulsar/client/impl/ProducerImpl.java | 29 ++-- .../pulsar/client/impl/TopicListWatcher.java | 27 +-- .../impl/TransactionMetaStoreHandler.java | 13 +- 6 files changed, 246 insertions(+), 53 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionHandlerTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionHandlerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionHandlerTest.java new file mode 100644 index 0000000000000..f29d62db5f480 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionHandlerTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.common.util.FutureUtil; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-impl") +public class ConnectionHandlerTest extends ProducerConsumerBase { + + private static final Backoff BACKOFF = new BackoffBuilder().setInitialTime(1, TimeUnit.MILLISECONDS) + .setMandatoryStop(1, TimeUnit.SECONDS) + .setMax(3, TimeUnit.SECONDS).create(); + private final ExecutorService executor = Executors.newFixedThreadPool(4); + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + executor.shutdown(); + } + + @Test(timeOut = 30000) + public void testSynchronousGrabCnx() { + for (int i = 0; i < 10; i++) { + final CompletableFuture future = new CompletableFuture<>(); + final int index = i; + final ConnectionHandler handler = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, "my-topic"), BACKOFF, + cnx -> { + future.complete(index); + return CompletableFuture.completedFuture(null); + }); + handler.grabCnx(); + Assert.assertEquals(future.join(), i); + } + } + + @Test + public void testConcurrentGrabCnx() { + final AtomicInteger cnt = new AtomicInteger(0); + final ConnectionHandler handler = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, "my-topic"), BACKOFF, + cnx -> { + cnt.incrementAndGet(); + return CompletableFuture.completedFuture(null); + }); + final int numGrab = 10; + for (int i = 0; i < numGrab; i++) { + handler.grabCnx(); + } + Awaitility.await().atMost(Duration.ofSeconds(3)).until(() -> cnt.get() > 0); + Assert.assertThrows(ConditionTimeoutException.class, + () -> Awaitility.await().atMost(Duration.ofMillis(500)).until(() -> cnt.get() == numGrab)); + Assert.assertEquals(cnt.get(), 1); + } + + @Test + public void testDuringConnectInvokeCount() throws IllegalAccessException { + // 1. connectionOpened completes with null + final AtomicBoolean duringConnect = spy(new AtomicBoolean()); + final ConnectionHandler handler1 = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, "my-topic"), BACKOFF, + cnx -> CompletableFuture.completedFuture(null)); + FieldUtils.writeField(handler1, "duringConnect", duringConnect, true); + handler1.grabCnx(); + Awaitility.await().atMost(Duration.ofSeconds(3)).until(() -> !duringConnect.get()); + verify(duringConnect, times(1)).compareAndSet(false, true); + verify(duringConnect, times(1)).set(false); + + // 2. connectionFailed is called + final ConnectionHandler handler2 = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, null), new MockedBackoff(), + cnx -> CompletableFuture.completedFuture(null)); + FieldUtils.writeField(handler2, "duringConnect", duringConnect, true); + handler2.grabCnx(); + Awaitility.await().atMost(Duration.ofSeconds(3)).until(() -> !duringConnect.get()); + verify(duringConnect, times(2)).compareAndSet(false, true); + verify(duringConnect, times(2)).set(false); + + // 3. connectionOpened completes exceptionally + final ConnectionHandler handler3 = new ConnectionHandler( + new MockedHandlerState((PulsarClientImpl) pulsarClient, "my-topic"), new MockedBackoff(), + cnx -> FutureUtil.failedFuture(new RuntimeException("fail"))); + FieldUtils.writeField(handler3, "duringConnect", duringConnect, true); + handler3.grabCnx(); + Awaitility.await().atMost(Duration.ofSeconds(3)).until(() -> !duringConnect.get()); + verify(duringConnect, times(3)).compareAndSet(false, true); + verify(duringConnect, times(3)).set(false); + } + + private static class MockedHandlerState extends HandlerState { + + public MockedHandlerState(PulsarClientImpl client, String topic) { + super(client, topic); + } + + @Override + String getHandlerName() { + return "mocked"; + } + } + + private static class MockedBackoff extends Backoff { + + // Set a large backoff so that reconnection won't happen in tests + public MockedBackoff() { + super(1, TimeUnit.HOURS, 2, TimeUnit.HOURS, 1, TimeUnit.HOURS); + } + } +} diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index 046cb90643a23..365abce3b90d2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -21,6 +21,7 @@ import java.net.InetSocketAddress; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLongFieldUpdater; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import org.apache.pulsar.client.api.PulsarClientException; @@ -41,10 +42,16 @@ public class ConnectionHandler { // Start with -1L because it gets incremented before sending on the first connection private volatile long epoch = -1L; protected volatile long lastConnectionClosedTimestamp = 0L; + private final AtomicBoolean duringConnect = new AtomicBoolean(false); interface Connection { - void connectionFailed(PulsarClientException exception); - void connectionOpened(ClientCnx cnx); + + /** + * @apiNote If the returned future is completed exceptionally, reconnectLater will be called. + */ + CompletableFuture connectionOpened(ClientCnx cnx); + default void connectionFailed(PulsarClientException e) { + } } protected Connection connection; @@ -69,6 +76,11 @@ protected void grabCnx() { state.topic, state.getHandlerName(), state.getState()); return; } + if (!duringConnect.compareAndSet(false, true)) { + log.info("[{}] [{}] Skip grabbing the connection since there is a pending connection", + state.topic, state.getHandlerName()); + return; + } try { CompletableFuture cnxFuture; @@ -81,7 +93,8 @@ protected void grabCnx() { } else { cnxFuture = state.client.getConnection(state.topic); // } - cnxFuture.thenAccept(cnx -> connection.connectionOpened(cnx)) // + cnxFuture.thenCompose(cnx -> connection.connectionOpened(cnx)) + .thenAccept(__ -> duringConnect.set(false)) .exceptionally(this::handleConnectionError); } catch (Throwable t) { log.warn("[{}] [{}] Exception thrown while getting connection: ", state.topic, state.getHandlerName(), t); @@ -90,25 +103,27 @@ protected void grabCnx() { } private Void handleConnectionError(Throwable exception) { - log.warn("[{}] [{}] Error connecting to broker: {}", - state.topic, state.getHandlerName(), exception.getMessage()); - if (exception instanceof PulsarClientException) { - connection.connectionFailed((PulsarClientException) exception); - } else if (exception.getCause() instanceof PulsarClientException) { - connection.connectionFailed((PulsarClientException) exception.getCause()); - } else { - connection.connectionFailed(new PulsarClientException(exception)); - } - - State state = this.state.getState(); - if (state == State.Uninitialized || state == State.Connecting || state == State.Ready) { - reconnectLater(exception); + try { + log.warn("[{}] [{}] Error connecting to broker: {}", + state.topic, state.getHandlerName(), exception.getMessage()); + if (exception instanceof PulsarClientException) { + connection.connectionFailed((PulsarClientException) exception); + } else if (exception.getCause() instanceof PulsarClientException) { + connection.connectionFailed((PulsarClientException) exception.getCause()); + } else { + connection.connectionFailed(new PulsarClientException(exception)); + } + } catch (Throwable throwable) { + log.error("[{}] [{}] Unexpected exception after the connection", + state.topic, state.getHandlerName(), throwable); } + reconnectLater(exception); return null; } - protected void reconnectLater(Throwable exception) { + void reconnectLater(Throwable exception) { + duringConnect.set(false); CLIENT_CNX_UPDATER.set(this, null); if (!isValidStateForReconnection()) { log.info("[{}] [{}] Ignoring reconnection request (state: {})", @@ -132,6 +147,7 @@ protected void reconnectLater(Throwable exception) { public void connectionClosed(ClientCnx cnx) { lastConnectionClosedTimestamp = System.currentTimeMillis(); + duringConnect.set(false); state.client.getCnxPool().releaseConnection(cnx); if (CLIENT_CNX_UPDATER.compareAndSet(this, cnx, null)) { if (!isValidStateForReconnection()) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 2dd245c10f55b..bb27562188d2f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -753,16 +753,17 @@ public void negativeAcknowledge(Message message) { } @Override - public void connectionOpened(final ClientCnx cnx) { + public CompletableFuture connectionOpened(final ClientCnx cnx) { previousExceptions.clear(); - if (getState() == State.Closing || getState() == State.Closed) { + final State state = getState(); + if (state == State.Closing || state == State.Closed) { setState(State.Closed); closeConsumerTasks(); deregisterFromClientCnx(); client.cleanupConsumer(this); clearReceiverQueue(); - return; + return CompletableFuture.completedFuture(null); } log.info("[{}][{}] Subscribing to topic on cnx {}, consumerId {}", @@ -816,6 +817,7 @@ public void connectionOpened(final ClientCnx cnx) { && startMessageId.equals(initialStartMessageId)) ? startMessageRollbackDurationInSec : 0; // synchronized this, because redeliverUnAckMessage eliminate the epoch inconsistency between them + final CompletableFuture future = new CompletableFuture<>(); synchronized (this) { setClientCnx(cnx); ByteBuf request = Commands.newSubscribe(topic, subscription, consumerId, requestId, getSubType(), @@ -837,6 +839,7 @@ public void connectionOpened(final ClientCnx cnx) { deregisterFromClientCnx(); client.cleanupConsumer(this); cnx.channel().close(); + future.complete(null); return; } } @@ -849,12 +852,14 @@ public void connectionOpened(final ClientCnx cnx) { if (!(firstTimeConnect && hasParentConsumer) && getCurrentReceiverQueueSize() != 0) { increaseAvailablePermits(cnx, getCurrentReceiverQueueSize()); } + future.complete(null); }).exceptionally((e) -> { deregisterFromClientCnx(); if (getState() == State.Closing || getState() == State.Closed) { // Consumer was closed while reconnecting, close the connection to make sure the broker // drops the consumer on its side cnx.channel().close(); + future.complete(null); return null; } log.warn("[{}][{}] Failed to subscribe to topic on {}", topic, @@ -872,7 +877,7 @@ public void connectionOpened(final ClientCnx cnx) { if (e.getCause() instanceof PulsarClientException && PulsarClientException.isRetriableError(e.getCause()) && System.currentTimeMillis() < SUBSCRIBE_DEADLINE_UPDATER.get(ConsumerImpl.this)) { - reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); } else if (!subscribeFuture.isDone()) { // unable to create new consumer, fail operation setState(State.Failed); @@ -896,11 +901,16 @@ public void connectionOpened(final ClientCnx cnx) { topic, subscription, cnx.channel().remoteAddress()); } else { // consumer was subscribed and connected but we got some error, keep trying - reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); + } + + if (!future.isDone()) { + future.complete(null); } return null; }); } + return future; } protected void consumerIsReconnectedToBroker(ClientCnx cnx, int currentQueueSize) { @@ -984,7 +994,7 @@ public void connectionFailed(PulsarClientException exception) { setState(State.Failed); if (nonRetriableError) { log.info("[{}] Consumer creation failed for consumer {} with unretriableError {}", - topic, consumerId, exception); + topic, consumerId, exception.getMessage()); } else { log.info("[{}] Consumer creation failed for consumer {} after timeout", topic, consumerId); } @@ -2583,10 +2593,6 @@ void deregisterFromClientCnx() { setClientCnx(null); } - void reconnectLater(Throwable exception) { - this.connectionHandler.reconnectLater(exception); - } - void grabCnx() { this.connectionHandler.grabCnx(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 2192ebfb64e75..267b06649d719 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -1649,8 +1649,9 @@ public Iterator iterator() { } } + @Override - public void connectionOpened(final ClientCnx cnx) { + public CompletableFuture connectionOpened(final ClientCnx cnx) { previousExceptions.clear(); chunkMaxMessageSize = Math.min(chunkMaxMessageSize, ClientCnx.getMaxMessageSize()); @@ -1659,7 +1660,7 @@ public void connectionOpened(final ClientCnx cnx) { // Because the state could have been updated while retrieving the connection, we set it back to connecting, // as long as the change from current state to connecting is a valid state change. if (!changeToConnecting()) { - return; + return CompletableFuture.completedFuture(null); } // We set the cnx reference before registering the producer on the cnx, so if the cnx breaks before creating // the producer, it will try to grab a new cnx. We also increment and get the epoch value for the producer. @@ -1699,6 +1700,7 @@ public void connectionOpened(final ClientCnx cnx) { } } + final CompletableFuture future = new CompletableFuture<>(); cnx.sendRequestWithId( Commands.newProducer(topic, producerId, requestId, producerName, conf.isEncryptionEnabled(), metadata, schemaInfo, epoch, userProvidedProducerName, @@ -1713,11 +1715,13 @@ public void connectionOpened(final ClientCnx cnx) { // We are now reconnected to broker and clear to send messages. Re-send all pending messages and // set the cnx pointer so that new messages will be sent immediately synchronized (ProducerImpl.this) { - if (getState() == State.Closing || getState() == State.Closed) { + State state = getState(); + if (state == State.Closing || state == State.Closed) { // Producer was closed while reconnecting, close the connection to make sure the broker // drops the producer on its side cnx.removeProducer(producerId); cnx.channel().close(); + future.complete(null); return; } resetBackoff(); @@ -1744,13 +1748,16 @@ public void connectionOpened(final ClientCnx cnx) { resendMessages(cnx, epoch); } + future.complete(null); }).exceptionally((e) -> { Throwable cause = e.getCause(); cnx.removeProducer(producerId); - if (getState() == State.Closing || getState() == State.Closed) { + State state = getState(); + if (state == State.Closing || state == State.Closed) { // Producer was closed while reconnecting, close the connection to make sure the broker // drops the producer on its side cnx.channel().close(); + future.complete(null); return null; } @@ -1779,6 +1786,7 @@ public void connectionOpened(final ClientCnx cnx) { } producerCreatedFuture.completeExceptionally(cause); }); + future.complete(null); return null; } if (cause instanceof PulsarClientException.ProducerBlockedQuotaExceededException) { @@ -1822,7 +1830,7 @@ public void connectionOpened(final ClientCnx cnx) { && System.currentTimeMillis() < PRODUCER_DEADLINE_UPDATER.get(ProducerImpl.this))) { // Either we had already created the producer once (producerCreatedFuture.isDone()) or we are // still within the initial timeout budget and we are dealing with a retriable error - reconnectLater(cause); + future.completeExceptionally(cause); } else { setState(State.Failed); producerCreatedFuture.completeExceptionally(cause); @@ -1834,9 +1842,12 @@ public void connectionOpened(final ClientCnx cnx) { sendTimeout = null; } } - + if (!future.isDone()) { + future.complete(null); + } return null; }); + return future; } @Override @@ -1848,7 +1859,7 @@ public void connectionFailed(PulsarClientException exception) { if (producerCreatedFuture.completeExceptionally(exception)) { if (nonRetriableError) { log.info("[{}] Producer creation failed for producer {} with unretriableError = {}", - topic, producerId, exception); + topic, producerId, exception.getMessage()); } else { log.info("[{}] Producer creation failed for producer {} after producerTimeout", topic, producerId); } @@ -2364,10 +2375,6 @@ void setClientCnx(ClientCnx clientCnx) { this.connectionHandler.setClientCnx(clientCnx); } - void reconnectLater(Throwable exception) { - this.connectionHandler.reconnectLater(exception); - } - void grabCnx() { this.connectionHandler.grabCnx(); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java index 384d1b688b8d5..8fe1e9bbf0ff8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java @@ -89,7 +89,7 @@ public void connectionFailed(PulsarClientException exception) { if (watcherFuture.completeExceptionally(exception)) { setState(State.Failed); log.info("[{}] Watcher creation failed for {} with non-retriable error {}", - topic, name, exception); + topic, name, exception.getMessage()); deregisterFromClientCnx(); } } else { @@ -98,13 +98,14 @@ public void connectionFailed(PulsarClientException exception) { } @Override - public void connectionOpened(ClientCnx cnx) { + public CompletableFuture connectionOpened(ClientCnx cnx) { previousExceptions.clear(); - if (getState() == State.Closing || getState() == State.Closed) { + State state = getState(); + if (state == State.Closing || state == State.Closed) { setState(State.Closed); deregisterFromClientCnx(); - return; + return CompletableFuture.completedFuture(null); } log.info("[{}][{}] Creating topic list watcher on cnx {}, watcherId {}", @@ -116,6 +117,7 @@ public void connectionOpened(ClientCnx cnx) { .compareAndSet(this, 0L, System.currentTimeMillis() + client.getConfiguration().getOperationTimeoutMs()); + final CompletableFuture future = new CompletableFuture<>(); // synchronized this, because redeliverUnAckMessage eliminate the epoch inconsistency between them synchronized (this) { setClientCnx(cnx); @@ -132,6 +134,7 @@ public void connectionOpened(ClientCnx cnx) { setState(State.Closed); deregisterFromClientCnx(); cnx.channel().close(); + future.complete(null); return; } } @@ -139,13 +142,14 @@ public void connectionOpened(ClientCnx cnx) { this.connectionHandler.resetBackoff(); watcherFuture.complete(this); - + future.complete(null); }).exceptionally((e) -> { deregisterFromClientCnx(); if (getState() == State.Closing || getState() == State.Closed) { // Watcher was closed while reconnecting, close the connection to make sure the broker // drops the watcher on its side cnx.channel().close(); + future.complete(null); return null; } log.warn("[{}][{}] Failed to create topic list watcher on {}", @@ -155,7 +159,7 @@ public void connectionOpened(ClientCnx cnx) { && PulsarClientException.isRetriableError(e.getCause()) && System.currentTimeMillis() < CREATE_WATCHER_DEADLINE_UPDATER.get(TopicListWatcher.this)) { - reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); } else if (!watcherFuture.isDone()) { // unable to create new watcher, fail operation setState(State.Failed); @@ -164,11 +168,15 @@ public void connectionOpened(ClientCnx cnx) { + "when connecting to the broker", getHandlerName()))); } else { // watcher was subscribed and connected, but we got some error, keep trying - reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); + } + if (!future.isDone()) { + future.complete(null); } return null; }); } + return future; } @Override @@ -249,11 +257,6 @@ void deregisterFromClientCnx() { setClientCnx(null); } - void reconnectLater(Throwable exception) { - this.connectionHandler.reconnectLater(exception); - } - - private void cleanupAtClose(CompletableFuture closeFuture, Throwable exception) { log.info("[{}] Closed topic list watcher", getHandlerName()); setState(State.Closed); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index 601fa2b8f815a..ebbfca0c3cb3f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -123,14 +123,17 @@ public void connectionFailed(PulsarClientException exception) { } @Override - public void connectionOpened(ClientCnx cnx) { + public CompletableFuture connectionOpened(ClientCnx cnx) { + final CompletableFuture future = new CompletableFuture<>(); internalPinnedExecutor.execute(() -> { LOG.info("Transaction meta handler with transaction coordinator id {} connection opened.", transactionCoordinatorId); - if (getState() == State.Closing || getState() == State.Closed) { + State state = getState(); + if (state == State.Closing || state == State.Closed) { setState(State.Closed); failPendingRequest(); + future.complete(null); return; } @@ -146,6 +149,7 @@ public void connectionOpened(ClientCnx cnx) { this.connectionHandler.resetBackoff(); pendingRequests.forEach((requestID, opBase) -> checkStateAndSendRequest(opBase)); } + future.complete(null); }); }).exceptionally((e) -> { internalPinnedExecutor.execute(() -> { @@ -155,16 +159,19 @@ public void connectionOpened(ClientCnx cnx) { || e.getCause() instanceof PulsarClientException.NotAllowedException) { setState(State.Closed); cnx.channel().close(); + future.complete(null); } else { - connectionHandler.reconnectLater(e.getCause()); + future.completeExceptionally(e.getCause()); } }); return null; }); } else { registerToConnection(cnx); + future.complete(null); } }); + return future; } private boolean registerToConnection(ClientCnx cnx) { From 4f8414a5cb8b266e8aab3046a37d78a570242837 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Thu, 29 Jun 2023 22:26:13 -0700 Subject: [PATCH 186/494] [fix][broker] Added the skipped message handler for ServiceUnitStateChannel (#20677) ### Motivation When topic lookups race(and the first lookup returns too fast before the second request gets deduped), the later lookup requests can timeout because the skipped messages are ignored at the table view and do not notify the service unit state channel. ex: m1: assign to b1 -> m2: owned by b1 -> m3: assign to b2 // m3 is skipped at the tableview When m3 is skipped, we better get the channel notified to return the lookup request with the current owner(b1) instead of letting the request time out. ### Modifications - Added the `handleSkippedMessage` and `setSkippedMsgHandler` in `TopicCompactionStrategy`. - `ServiceUnitStateChannel` registers `ServiceUnitStateChannel.handleSkippedEvent` to `ServiceUnitStateCompactionStrategy` by `setSkippedMsgHandler`. - TableView calls `TopicCompactionStrategy.handleSkippedMessage` when messages are skipped. --- .../channel/ServiceUnitStateChannelImpl.java | 21 +++++++++++++++++++ .../ServiceUnitStateCompactionStrategy.java | 13 ++++++++++++ .../channel/ServiceUnitStateChannelTest.java | 6 +++--- .../TopicCompactionStrategyTest.java | 4 ++-- .../pulsar/client/impl/TableViewImpl.java | 5 ++++- .../topics/TopicCompactionStrategy.java | 21 ++++++++++++++++--- 6 files changed, 61 insertions(+), 9 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index fbf3534e7d440..70fb8f9ba6f89 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -42,6 +42,7 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; +import static org.apache.pulsar.common.topics.TopicCompactionStrategy.TABLE_VIEW_TAG; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionReestablished; import com.google.common.annotations.VisibleForTesting; @@ -94,6 +95,7 @@ import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.common.topics.TopicCompactionStrategy; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -321,6 +323,13 @@ public synchronized void start() throws PulsarServerException { ServiceUnitStateCompactionStrategy.class.getName())) .create(); tableview.listen((key, value) -> handle(key, value)); + var strategy = (ServiceUnitStateCompactionStrategy) TopicCompactionStrategy.getInstance(TABLE_VIEW_TAG); + if (strategy == null) { + String err = TABLE_VIEW_TAG + "tag TopicCompactionStrategy is null."; + log.error(err); + throw new IllegalStateException(err); + } + strategy.setSkippedMsgHandler((key, value) -> handleSkippedEvent(key)); if (debug) { log.info("Successfully started the channel tableview."); } @@ -695,6 +704,18 @@ lookupServiceAddress, getLogEventTag(data), serviceUnit, } } + private void handleSkippedEvent(String serviceUnit) { + var getOwnerRequest = getOwnerRequests.get(serviceUnit); + if (getOwnerRequest != null) { + var data = tableview.get(serviceUnit); + if (data.state() == Owned) { + getOwnerRequest.complete(data.dstBroker()); + getOwnerRequests.remove(serviceUnit); + stateChangeListeners.notify(serviceUnit, data, null); + } + } + } + private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { var getOwnerRequest = getOwnerRequests.remove(serviceUnit); if (getOwnerRequest != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java index 72b05b5cd62c8..6a98b79be81d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateCompactionStrategy.java @@ -22,6 +22,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; import com.google.common.annotations.VisibleForTesting; +import java.util.function.BiConsumer; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.topics.TopicCompactionStrategy; @@ -29,6 +30,7 @@ public class ServiceUnitStateCompactionStrategy implements TopicCompactionStrategy { private final Schema schema; + private BiConsumer skippedMsgHandler; private boolean checkBrokers = true; @@ -36,6 +38,17 @@ public ServiceUnitStateCompactionStrategy() { schema = Schema.JSON(ServiceUnitStateData.class); } + public void setSkippedMsgHandler(BiConsumer skippedMsgHandler) { + this.skippedMsgHandler = skippedMsgHandler; + } + + @Override + public void handleSkippedMessage(String key, ServiceUnitStateData cur) { + if (skippedMsgHandler != null) { + skippedMsgHandler.accept(key, cur); + } + } + @Override public Schema getSchema() { return schema; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 1263170bec495..d8b4f1d75d287 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -53,6 +53,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNull; import java.lang.reflect.Field; import java.util.List; import java.util.Map; @@ -199,7 +200,7 @@ protected void cleanup() throws Exception { super.internalCleanup(); } - @Test(priority = 0) + @Test(priority = -1) public void channelOwnerTest() throws Exception { var channelOwner1 = channel1.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); var channelOwner2 = channel2.getChannelOwnerAsync().get(2, TimeUnit.SECONDS).get(); @@ -947,8 +948,7 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx } catch (CompletionException e) { ex = e; } - assertNotNull(ex); - assertEquals(TimeoutException.class, ex.getCause().getClass()); + assertNull(ex); assertEquals(Optional.of(lookupServiceAddress1), channel2.getOwnerAsync(bundle).get()); assertEquals(Optional.of(lookupServiceAddress1), channel1.getOwnerAsync(bundle).get()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionStrategyTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionStrategyTest.java index 0ecd09606cea7..50d4cdc7b0a71 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionStrategyTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/TopicCompactionStrategyTest.java @@ -41,13 +41,13 @@ public boolean shouldKeepLeft(byte[] prev, byte[] cur) { @Test(expectedExceptions = IllegalArgumentException.class) public void testLoadInvalidTopicCompactionStrategy() { - TopicCompactionStrategy.load("uknown"); + TopicCompactionStrategy.load("uknown", "uknown"); } @Test public void testNumericOrderCompactionStrategy() { TopicCompactionStrategy strategy = - TopicCompactionStrategy.load(NumericOrderCompactionStrategy.class.getCanonicalName()); + TopicCompactionStrategy.load("numeric", NumericOrderCompactionStrategy.class.getCanonicalName()); Assert.assertFalse(strategy.shouldKeepLeft(1, 2)); Assert.assertTrue(strategy.shouldKeepLeft(2, 1)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java index 77aba7e48cbad..560636f94622b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import static org.apache.pulsar.common.topics.TopicCompactionStrategy.TABLE_VIEW_TAG; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -65,7 +66,8 @@ public class TableViewImpl implements TableView { this.immutableData = Collections.unmodifiableMap(data); this.listeners = new ArrayList<>(); this.listenersMutex = new ReentrantLock(); - this.compactionStrategy = TopicCompactionStrategy.load(conf.getTopicCompactionStrategyClassName()); + this.compactionStrategy = + TopicCompactionStrategy.load(TABLE_VIEW_TAG, conf.getTopicCompactionStrategyClassName()); ReaderBuilder readerBuilder = client.newReader(schema) .topic(conf.getTopicName()) .startMessageId(MessageId.earliest) @@ -198,6 +200,7 @@ private void handleMessage(Message msg) { key, cur, prev); + compactionStrategy.handleSkippedMessage(key, cur); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java index f06374b234fc7..51fb6742961bb 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/topics/TopicCompactionStrategy.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.common.topics; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.pulsar.client.api.Schema; /** @@ -45,6 +47,9 @@ */ public interface TopicCompactionStrategy { + String TABLE_VIEW_TAG = "table-view"; + Map INSTANCES = new ConcurrentHashMap<>(); + /** * Returns the schema object for this strategy. * @return @@ -60,17 +65,27 @@ public interface TopicCompactionStrategy { */ boolean shouldKeepLeft(T prev, T cur); - static TopicCompactionStrategy load(String topicCompactionStrategyClassName) { + default void handleSkippedMessage(String key, T cur) { + } + + + static TopicCompactionStrategy load(String tag, String topicCompactionStrategyClassName) { if (topicCompactionStrategyClassName == null) { return null; } + try { Class clazz = Class.forName(topicCompactionStrategyClassName); - Object instance = clazz.getDeclaredConstructor().newInstance(); - return (TopicCompactionStrategy) instance; + TopicCompactionStrategy instance = (TopicCompactionStrategy) clazz.getDeclaredConstructor().newInstance(); + INSTANCES.put(tag, instance); + return instance; } catch (Exception e) { throw new IllegalArgumentException( "Error when loading topic compaction strategy: " + topicCompactionStrategyClassName, e); } } + + static TopicCompactionStrategy getInstance(String tag) { + return INSTANCES.get(tag); + } } From e407c4b86a64e4eaa5ff291048d03c90cfb71035 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Fri, 16 Jun 2023 09:11:59 +0800 Subject: [PATCH 187/494] [improve][test] Add integration test for ExtensibleLoadManager (#20138) --- .github/workflows/pulsar-ci.yaml | 4 + build/run_integration_group.sh | 4 + .../containers/BrokerContainer.java | 4 + .../containers/PulsarContainer.java | 2 +- .../ExtensibleLoadManagerTest.java | 390 ++++++++++++++++++ .../src/test/resources/pulsar-loadbalance.xml | 28 ++ .../integration/src/test/resources/pulsar.xml | 1 + 7 files changed, 432 insertions(+), 1 deletion(-) create mode 100644 tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java create mode 100644 tests/integration/src/test/resources/pulsar-loadbalance.xml diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 7139ea4cff910..35182f0bbd837 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -478,6 +478,10 @@ jobs: - name: Messaging group: MESSAGING + - name: LoadBalance + group: LOADBALANCE + no_coverage: true + - name: Shade on Java 8 group: SHADE_RUN runtime_jdk: 8 diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh index bc1255d8d68aa..9372d1ecb9317 100755 --- a/build/run_integration_group.sh +++ b/build/run_integration_group.sh @@ -156,6 +156,10 @@ test_group_messaging() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-tls.xml -DintegrationTests } +test_group_loadbalance() { + mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-loadbalance.xml -DintegrationTests +} + test_group_plugin() { mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-plugin.xml -DintegrationTests } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BrokerContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BrokerContainer.java index e0e85af024bc2..616d45554d75c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BrokerContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/BrokerContainer.java @@ -33,6 +33,10 @@ public BrokerContainer(String clusterName, String hostName) { tailContainerLog(); } + public String getHostName() { + return super.hostname; + } + @Override protected void afterStart() { DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(), diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java index dc11acd00c3f2..56d64ce5b2c8e 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PulsarContainer.java @@ -70,7 +70,7 @@ public abstract class PulsarContainer> exte public static final boolean PULSAR_CONTAINERS_LEAVE_RUNNING = Boolean.parseBoolean(System.getenv("PULSAR_CONTAINERS_LEAVE_RUNNING")); - private final String hostname; + protected final String hostname; private final String serviceName; private final String serviceEntryPoint; private final int servicePort; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java new file mode 100644 index 0000000000000..5fd2cba4153cd --- /dev/null +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.tests.integration.loadbalance; + +import static org.apache.pulsar.tests.integration.containers.PulsarContainer.BROKER_HTTP_PORT; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import com.google.common.collect.Sets; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData; +import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType; +import org.apache.pulsar.common.policies.data.BundlesData; +import org.apache.pulsar.common.policies.data.FailureDomain; +import org.apache.pulsar.common.policies.data.NamespaceIsolationData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.tests.integration.containers.BrokerContainer; +import org.apache.pulsar.tests.integration.topologies.PulsarCluster; +import org.apache.pulsar.tests.integration.topologies.PulsarClusterSpec; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * Integration tests for Pulsar ExtensibleLoadManagerImpl. + */ +@Slf4j +public class ExtensibleLoadManagerTest extends TestRetrySupport { + + private static final int NUM_BROKERS = 3; + private static final String DEFAULT_TENANT = "my-tenant"; + private static final String DEFAULT_NAMESPACE = DEFAULT_TENANT + "/my-namespace"; + private static final String nsSuffix = "-anti-affinity-enabled"; + + private final String clusterName = "MultiLoadManagerTest-" + UUID.randomUUID(); + private final PulsarClusterSpec spec = PulsarClusterSpec.builder() + .clusterName(clusterName) + .numBrokers(NUM_BROKERS).build(); + private PulsarCluster pulsarCluster = null; + private List brokerUrls = null; + private String hosts; + private PulsarAdmin admin; + + @BeforeClass(alwaysRun = true) + public void setup() throws Exception { + incrementSetupNumber(); + Map brokerEnvs = new HashMap<>(); + brokerEnvs.put("loadManagerClassName", + "org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl"); + brokerEnvs.put("loadBalancerLoadSheddingStrategy", + "org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder"); + brokerEnvs.put("forceDeleteNamespaceAllowed", "true"); + brokerEnvs.put("loadBalancerDebugModeEnabled", "true"); + brokerEnvs.put("PULSAR_MEM", "-Xmx512M"); + spec.brokerEnvs(brokerEnvs); + pulsarCluster = PulsarCluster.forSpec(spec); + pulsarCluster.start(); + brokerUrls = brokerUrls(); + + hosts = pulsarCluster.getAllBrokersHttpServiceUrl(); + admin = PulsarAdmin.builder().serviceHttpUrl(hosts).build(); + // all brokers alive + assertEquals(admin.brokers().getActiveBrokers(clusterName).size(), NUM_BROKERS); + + admin.tenants().createTenant(DEFAULT_TENANT, + new TenantInfoImpl(new HashSet<>(), Set.of(pulsarCluster.getClusterName()))); + admin.namespaces().createNamespace(DEFAULT_NAMESPACE, 100); + } + + @AfterClass(alwaysRun = true) + public void cleanup() { + markCurrentSetupNumberCleaned(); + if (pulsarCluster != null) { + pulsarCluster.stop(); + pulsarCluster = null; + } + } + + @BeforeMethod(alwaysRun = true) + public void startBroker() { + if (pulsarCluster != null) { + pulsarCluster.getBrokers().forEach(brokerContainer -> { + if (!brokerContainer.isRunning()) { + brokerContainer.start(); + } + }); + } + } + + @Test(timeOut = 40 * 1000) + public void testConcurrentLookups() throws Exception { + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/testConcurrentLookups"; + List admins = new ArrayList<>(); + int numAdminForBroker = 10; + for (String url : brokerUrls) { + for (int i = 0; i < numAdminForBroker; i++) { + admins.add(PulsarAdmin.builder().serviceHttpUrl(url).build()); + } + } + + admin.topics().createPartitionedTopic(topicName, 100); + + var executor = Executors.newFixedThreadPool(admins.size()); + + CountDownLatch latch = new CountDownLatch(admins.size()); + List> result = new CopyOnWriteArrayList<>(); + for(var admin : admins) { + executor.execute(() -> { + try { + result.add(admin.lookups().lookupPartitionedTopic(topicName)); + } catch (PulsarAdminException e) { + log.error("Lookup partitioned topic failed.", e); + } + latch.countDown(); + }); + } + latch.await(); + + assertEquals(result.size(), admins.size()); + + for (int i = 1; i < admins.size(); i++) { + assertEquals(result.get(i - 1), result.get(i)); + } + } + + @Test(timeOut = 30 * 1000) + public void testTransferAdminApi() throws Exception { + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/testUnloadAdminApi"; + admin.topics().createNonPartitionedTopic(topicName); + String broker = admin.lookups().lookupTopic(topicName); + + int index = extractBrokerIndex(broker); + + String bundleRange = admin.lookups().getBundleRange(topicName); + + // Test transfer to current broker. + try { + admin.namespaces().unloadNamespaceBundle(DEFAULT_NAMESPACE, bundleRange, getBrokerUrl(index)); + fail(); + } catch (PulsarAdminException ex) { + assertTrue(ex.getMessage().contains("cannot be transfer to same broker")); + } + + int transferToIndex = generateRandomExcludingX(NUM_BROKERS, index); + assertNotEquals(transferToIndex, index); + String transferTo = getBrokerUrl(transferToIndex); + admin.namespaces().unloadNamespaceBundle(DEFAULT_NAMESPACE, bundleRange, transferTo); + + broker = admin.lookups().lookupTopic(topicName); + + index = extractBrokerIndex(broker); + assertEquals(index, transferToIndex); + } + + @Test(timeOut = 30 * 1000) + public void testSplitBundleAdminApi() throws Exception { + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/testSplitBundleAdminApi"; + admin.topics().createNonPartitionedTopic(topicName); + String broker = admin.lookups().lookupTopic(topicName); + log.info("The topic: {} owned by {}", topicName, broker); + BundlesData bundles = admin.namespaces().getBundles(DEFAULT_NAMESPACE); + int numBundles = bundles.getNumBundles(); + var bundleRanges = bundles.getBoundaries().stream().map(Long::decode).sorted().toList(); + String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1); + admin.namespaces().splitNamespaceBundle(DEFAULT_NAMESPACE, firstBundle, true, null); + BundlesData bundlesData = admin.namespaces().getBundles(DEFAULT_NAMESPACE); + + long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2; + + assertEquals(bundlesData.getNumBundles(), numBundles + 1); + String lowBundle = String.format("0x%08x", bundleRanges.get(0)); + String midBundle = String.format("0x%08x", mid); + String highBundle = String.format("0x%08x", bundleRanges.get(1)); + assertTrue(bundlesData.getBoundaries().contains(lowBundle)); + assertTrue(bundlesData.getBoundaries().contains(midBundle)); + assertTrue(bundlesData.getBoundaries().contains(highBundle)); + + // Test split bundle with invalid bundle range. + try { + admin.namespaces().splitNamespaceBundle(DEFAULT_NAMESPACE, "invalid", true, null); + fail(); + } catch (PulsarAdminException ex) { + assertTrue(ex.getMessage().contains("Invalid bundle range")); + } + } + + @Test(timeOut = 30 * 1000) + public void testDeleteNamespace() throws Exception { + String namespace = DEFAULT_TENANT + "/test-delete-namespace"; + String topicName = "persistent://" + namespace + "/test-delete-namespace-topic"; + admin.namespaces().createNamespace(namespace); + admin.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet(clusterName)); + assertTrue(admin.namespaces().getNamespaces(DEFAULT_TENANT).contains(namespace)); + admin.topics().createPartitionedTopic(topicName, 2); + String broker = admin.lookups().lookupTopic(topicName); + log.info("The topic: {} owned by: {}", topicName, broker); + admin.namespaces().deleteNamespace(namespace, true); + assertFalse(admin.namespaces().getNamespaces(DEFAULT_TENANT).contains(namespace)); + } + + @Test(timeOut = 40 * 1000) + public void testStopBroker() throws Exception { + String topicName = "persistent://" + DEFAULT_NAMESPACE + "/test-stop-broker-topic"; + + admin.topics().createNonPartitionedTopic(topicName); + String broker = admin.lookups().lookupTopic(topicName); + log.info("The topic: {} owned by: {}", topicName, broker); + + int idx = extractBrokerIndex(broker); + for (BrokerContainer container : pulsarCluster.getBrokers()) { + String name = container.getHostName(); + if (name.contains(String.valueOf(idx))) { + container.stop(); + } + } + + String broker1 = admin.lookups().lookupTopic(topicName); + + assertNotEquals(broker1, broker); + } + + @Test(timeOut = 40 * 1000) + public void testAntiaffinityPolicy() throws PulsarAdminException { + final String namespaceAntiAffinityGroup = "my-anti-affinity-filter"; + final String antiAffinityEnabledNameSpace = DEFAULT_TENANT + "/my-ns-filter" + nsSuffix; + final int numPartition = 20; + + List activeBrokers = admin.brokers().getActiveBrokers(); + + assertEquals(activeBrokers.size(), NUM_BROKERS); + + for (int i = 0; i < activeBrokers.size(); i++) { + String namespace = antiAffinityEnabledNameSpace + "-" + i; + admin.namespaces().createNamespace(namespace, 10); + admin.namespaces().setNamespaceAntiAffinityGroup(namespace, namespaceAntiAffinityGroup); + admin.clusters().createFailureDomain(clusterName, namespaceAntiAffinityGroup, FailureDomain.builder() + .brokers(Set.of(activeBrokers.get(i))).build()); + } + + Set result = new HashSet<>(); + for (int i = 0; i < activeBrokers.size(); i++) { + final String topic = "persistent://" + antiAffinityEnabledNameSpace + "-" + i +"/topic"; + admin.topics().createPartitionedTopic(topic, numPartition); + + Map topicToBroker = admin.lookups().lookupPartitionedTopic(topic); + + assertEquals(topicToBroker.size(), numPartition); + + HashSet brokers = new HashSet<>(topicToBroker.values()); + + assertEquals(brokers.size(), 1); + result.add(brokers.iterator().next()); + log.info("Topic: {}, lookup result: {}", topic, brokers.iterator().next()); + } + + assertEquals(result.size(), NUM_BROKERS); + } + + @Test(timeOut = 40 * 1000) + public void testIsolationPolicy() throws PulsarAdminException { + final String namespaceIsolationPolicyName = "my-isolation-policy"; + final String isolationEnabledNameSpace = DEFAULT_TENANT + "/my-isolation-policy" + nsSuffix; + Map parameters1 = new HashMap<>(); + parameters1.put("min_limit", "1"); + parameters1.put("usage_threshold", "100"); + + List activeBrokers = admin.brokers().getActiveBrokers(); + + assertEquals(activeBrokers.size(), NUM_BROKERS); + + admin.namespaces().createNamespace(isolationEnabledNameSpace); + admin.clusters().createNamespaceIsolationPolicy(clusterName, namespaceIsolationPolicyName, NamespaceIsolationData + .builder() + .namespaces(List.of(isolationEnabledNameSpace)) + .autoFailoverPolicy(AutoFailoverPolicyData.builder() + .policyType(AutoFailoverPolicyType.min_available) + .parameters(parameters1) + .build()) + .primary(List.of(getHostName(0))) + .secondary(List.of(getHostName(1))) + .build()); + final String topic = "persistent://" + isolationEnabledNameSpace + "/topic"; + admin.topics().createNonPartitionedTopic(topic); + + String broker = admin.lookups().lookupTopic(topic); + + for (BrokerContainer container : pulsarCluster.getBrokers()) { + String name = container.getHostName(); + if (name.contains("0")) { + container.stop(); + } + } + + assertEquals(extractBrokerIndex(broker), 0); + + broker = admin.lookups().lookupTopic(topic); + + assertEquals(extractBrokerIndex(broker), 1); + + for (BrokerContainer container : pulsarCluster.getBrokers()) { + String name = container.getHostName(); + if (name.contains("1")) { + container.stop(); + } + } + try { + admin.lookups().lookupTopic(topic); + fail(); + } catch (Exception ex) { + log.error("Failed to lookup topic: ", ex); + assertTrue(ex.getMessage().contains("Failed to look up a broker")); + } + } + + private String getBrokerUrl(int index) { + return String.format("pulsar-broker-%d:%d", index, BROKER_HTTP_PORT); + } + + private String getHostName(int index) { + return String.format("pulsar-broker-%d", index); + } + + private int extractBrokerIndex(String broker) { + String pattern = "pulsar://.*-(\\d+):\\d+"; + Pattern compiledPattern = Pattern.compile(pattern); + Matcher matcher = compiledPattern.matcher(broker); + if (!matcher.find()){ + throw new IllegalArgumentException("Failed to extract broker index"); + } + return Integer.parseInt(matcher.group(1)); + } + + private int generateRandomExcludingX(int n, int x) { + Random random = new Random(); + int randomNumber; + + do { + randomNumber = random.nextInt(n); + } while (randomNumber == x); + + return randomNumber; + } + + private List brokerUrls() { + Collection brokers = pulsarCluster.getBrokers(); + List brokerUrls = new ArrayList<>(NUM_BROKERS); + brokers.forEach(broker -> { + brokerUrls.add("http://" + broker.getHost() + ":" + broker.getMappedPort(BROKER_HTTP_PORT)); + }); + return brokerUrls; + } +} diff --git a/tests/integration/src/test/resources/pulsar-loadbalance.xml b/tests/integration/src/test/resources/pulsar-loadbalance.xml new file mode 100644 index 0000000000000..dfc4536e25592 --- /dev/null +++ b/tests/integration/src/test/resources/pulsar-loadbalance.xml @@ -0,0 +1,28 @@ + + + + + + + + + \ No newline at end of file diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index 7fbe9fa24d8d9..d309a7695dd0b 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -30,6 +30,7 @@ + From 00ff0903c3147a1728193421ae18f0dab3fa626e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 20 Jun 2023 16:05:23 +0300 Subject: [PATCH 188/494] [fix][test] Fix flaky test ExtensibleLoadManagerTest.testIsolationPolicy (#20609) --- pom.xml | 2 +- .../ExtensibleLoadManagerTest.java | 40 +++++++++++-------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index 9617efcdc6dc4..b63af0581805f 100644 --- a/pom.xml +++ b/pom.xml @@ -222,7 +222,7 @@ flexible messaging model and an intuitive client API. 3.21.0 0.9.1 2.1.0 - 3.18.1 + 3.24.2 1.18.26 1.3.2 2.3.1 diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 5fd2cba4153cd..86228d25a793f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -19,13 +19,27 @@ package org.apache.pulsar.tests.integration.loadbalance; import static org.apache.pulsar.tests.integration.containers.PulsarContainer.BROKER_HTTP_PORT; +import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; - import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -43,20 +57,6 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** @@ -112,6 +112,10 @@ public void cleanup() { pulsarCluster.stop(); pulsarCluster = null; } + if (admin != null) { + admin.close(); + admin = null; + } } @BeforeMethod(alwaysRun = true) @@ -257,7 +261,8 @@ public void testStopBroker() throws Exception { assertNotEquals(broker1, broker); } - @Test(timeOut = 40 * 1000) + // TODO: This test is very flaky and it's disabled for now to unblock CI + @Test(timeOut = 40 * 1000, enabled = false) public void testAntiaffinityPolicy() throws PulsarAdminException { final String namespaceAntiAffinityGroup = "my-anti-affinity-filter"; final String antiAffinityEnabledNameSpace = DEFAULT_TENANT + "/my-ns-filter" + nsSuffix; @@ -346,7 +351,8 @@ public void testIsolationPolicy() throws PulsarAdminException { fail(); } catch (Exception ex) { log.error("Failed to lookup topic: ", ex); - assertTrue(ex.getMessage().contains("Failed to look up a broker")); + assertThat(ex.getMessage()).containsAnyOf("Failed to look up a broker", + "Failed to select the new owner broker for bundle"); } } From 172550e33a18df53639497afb26a3195956b772b Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Mon, 3 Jul 2023 19:59:09 +0800 Subject: [PATCH 189/494] [improve][broker] Make ExtensibleLoadManagerImpl's broker filter pure async (#20666) --- .../extensions/ExtensibleLoadManagerImpl.java | 38 +++--- .../channel/ServiceUnitStateChannelImpl.java | 4 +- .../filter/AntiAffinityGroupPolicyFilter.java | 6 +- .../extensions/filter/BrokerFilter.java | 9 +- .../filter/BrokerIsolationPoliciesFilter.java | 19 ++- .../filter/BrokerLoadManagerClassFilter.java | 11 +- .../filter/BrokerMaxTopicCountFilter.java | 10 +- .../filter/BrokerVersionFilter.java | 17 +-- .../AntiAffinityGroupPolicyHelper.java | 11 +- .../policies/IsolationPoliciesHelper.java | 19 +-- .../extensions/scheduler/TransferShedder.java | 7 +- .../loadbalance/impl/LoadManagerShared.java | 114 ++++++++++++++++++ .../SimpleResourceAllocationPolicies.java | 15 +++ ...tiAffinityNamespaceGroupExtensionTest.java | 4 +- .../ExtensibleLoadManagerImplTest.java | 36 +++--- .../BrokerIsolationPoliciesFilterTest.java | 33 ++--- .../BrokerLoadManagerClassFilterTest.java | 7 +- .../filter/BrokerMaxTopicCountFilterTest.java | 6 +- .../filter/BrokerVersionFilterTest.java | 29 +++-- .../scheduler/TransferShedderTest.java | 27 +++-- .../ExtensibleLoadManagerTest.java | 3 +- 21 files changed, 282 insertions(+), 143 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index d0c55a6519159..f3c5e5dc2ee36 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -46,7 +47,6 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; @@ -477,10 +477,10 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, Set excludeBrokerSet) { BrokerRegistry brokerRegistry = getBrokerRegistry(); return brokerRegistry.getAvailableBrokerLookupDataAsync() - .thenCompose(availableBrokers -> { + .thenComposeAsync(availableBrokers -> { LoadManagerContext context = this.getContext(); - Map availableBrokerCandidates = new HashMap<>(availableBrokers); + Map availableBrokerCandidates = new ConcurrentHashMap<>(availableBrokers); if (!excludeBrokerSet.isEmpty()) { for (String exclude : excludeBrokerSet) { availableBrokerCandidates.remove(exclude); @@ -489,24 +489,28 @@ public CompletableFuture> selectAsync(ServiceUnitId bundle, // Filter out brokers that do not meet the rules. List filterPipeline = getBrokerFilterPipeline(); + ArrayList>> futures = + new ArrayList<>(filterPipeline.size()); for (final BrokerFilter filter : filterPipeline) { - try { - filter.filter(availableBrokerCandidates, bundle, context); - // Preserve the filter successes result. - availableBrokers.keySet().retainAll(availableBrokerCandidates.keySet()); - } catch (BrokerFilterException e) { + CompletableFuture> future = + filter.filter(availableBrokerCandidates, bundle, context); + futures.add(future); + } + CompletableFuture> result = new CompletableFuture<>(); + FutureUtil.waitForAll(futures).whenComplete((__, ex) -> { + if (ex != null) { // TODO: We may need to revisit this error case. - log.error("Failed to filter out brokers.", e); - availableBrokerCandidates = new HashMap<>(availableBrokers); + log.error("Failed to filter out brokers when select bundle: {}", bundle, ex); } - } - if (availableBrokerCandidates.isEmpty()) { - return CompletableFuture.completedFuture(Optional.empty()); - } - Set candidateBrokers = availableBrokerCandidates.keySet(); + if (availableBrokerCandidates.isEmpty()) { + result.complete(Optional.empty()); + return; + } + Set candidateBrokers = availableBrokerCandidates.keySet(); - return CompletableFuture.completedFuture( - getBrokerSelectionStrategy().select(candidateBrokers, bundle, context)); + result.complete(getBrokerSelectionStrategy().select(candidateBrokers, bundle, context)); + }); + return result; }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 70fb8f9ba6f89..717ff484fe772 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -619,7 +619,7 @@ public CompletableFuture publishSplitEventAsync(Split split) { private void handle(String serviceUnit, ServiceUnitStateData data) { long totalHandledRequests = getHandlerTotalCounter(data).incrementAndGet(); - if (log.isDebugEnabled()) { + if (debug()) { log.info("{} received a handle request for serviceUnit:{}, data:{}. totalHandledRequests:{}", lookupServiceAddress, serviceUnit, data, totalHandledRequests); } @@ -708,7 +708,7 @@ private void handleSkippedEvent(String serviceUnit) { var getOwnerRequest = getOwnerRequests.get(serviceUnit); if (getOwnerRequest != null) { var data = tableview.get(serviceUnit); - if (data.state() == Owned) { + if (data != null && data.state() == Owned) { getOwnerRequest.complete(data.dstBroker()); getOwnerRequests.remove(serviceUnit); stateChangeListeners.notify(serviceUnit, data, null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java index 462f8f0e3597a..df08ba96b9d7a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/AntiAffinityGroupPolicyFilter.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.AntiAffinityGroupPolicyHelper; @@ -38,10 +39,9 @@ public AntiAffinityGroupPolicyFilter(AntiAffinityGroupPolicyHelper helper) { } @Override - public Map filter( + public CompletableFuture> filter( Map brokers, ServiceUnitId serviceUnitId, LoadManagerContext context) { - helper.filter(brokers, serviceUnitId.toString()); - return brokers; + return helper.filterAsync(brokers, serviceUnitId.toString()); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java index d9cbfdc391ed4..046cae48914f8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerFilter.java @@ -19,7 +19,7 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -42,9 +42,8 @@ public interface BrokerFilter { * @param context The load manager context. * @return Filtered broker list. */ - Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException; + CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java index eeb0d9d3a3309..0aa1dda437af7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilter.java @@ -19,9 +19,8 @@ package org.apache.pulsar.broker.loadbalance.extensions.filter; import java.util.Map; -import java.util.Set; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.policies.IsolationPoliciesHelper; @@ -45,13 +44,13 @@ public String name() { } @Override - public Map filter(Map availableBrokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { - Set brokerCandidateCache = - isolationPoliciesHelper.applyIsolationPolicies(availableBrokers, serviceUnit); - availableBrokers.keySet().retainAll(brokerCandidateCache); - return availableBrokers; + public CompletableFuture> filter(Map availableBrokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return isolationPoliciesHelper.applyIsolationPoliciesAsync(availableBrokers, serviceUnit) + .thenApply(brokerCandidateCache -> { + availableBrokers.keySet().retainAll(brokerCandidateCache); + return availableBrokers; + }); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java index 07109b277ae98..6504da047a001 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilter.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.Objects; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -34,13 +34,12 @@ public String name() { } @Override - public Map filter( + public CompletableFuture> filter( Map brokers, ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { + LoadManagerContext context) { if (brokers.isEmpty()) { - return brokers; + return CompletableFuture.completedFuture(brokers); } brokers.entrySet().removeIf(entry -> { BrokerLookupData v = entry.getValue(); @@ -48,6 +47,6 @@ public Map filter( return !Objects.equals(v.getLoadManagerClassName(), context.brokerConfiguration().getLoadManagerClassName()); }); - return brokers; + return CompletableFuture.completedFuture(brokers); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java index 0bceae36bb8c2..044dcc83a7835 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilter.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.Optional; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; @@ -36,9 +36,9 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { int loadBalancerBrokerMaxTopics = context.brokerConfiguration().getLoadBalancerBrokerMaxTopics(); brokers.keySet().removeIf(broker -> { Optional brokerLoadDataOpt = context.brokerLoadDataStore().get(broker); @@ -46,6 +46,6 @@ public Map filter(Map broker // TODO: The broker load data might be delayed, so the max topic check might not accurate. return topics >= loadBalancerBrokerMaxTopics; }); - return brokers; + return CompletableFuture.completedFuture(brokers); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java index 7420fcc211309..6e8ce5e58f2bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilter.java @@ -21,13 +21,14 @@ import com.github.zafarkhaja.semver.Version; import java.util.Iterator; import java.util.Map; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.util.FutureUtil; /** * Filter by broker version. @@ -46,13 +47,12 @@ public class BrokerVersionFilter implements BrokerFilter { * */ @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) - throws BrokerFilterException { + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { ServiceConfiguration conf = context.brokerConfiguration(); if (!conf.isPreferLaterVersions() || brokers.isEmpty()) { - return brokers; + return CompletableFuture.completedFuture(brokers); } Version latestVersion; @@ -63,7 +63,8 @@ public Map filter(Map broker } } catch (Exception ex) { log.warn("Disabling PreferLaterVersions feature; reason: " + ex.getMessage()); - throw new BrokerFilterBadVersionException("Cannot determine newest broker version: " + ex.getMessage()); + return FutureUtil.failedFuture( + new BrokerFilterBadVersionException("Cannot determine newest broker version: " + ex.getMessage())); } int numBrokersLatestVersion = 0; @@ -88,7 +89,7 @@ public Map filter(Map broker if (numBrokersOlderVersion == 0) { log.info("All {} brokers are running the latest version [{}]", numBrokersLatestVersion, latestVersion); } - return brokers; + return CompletableFuture.completedFuture(brokers); } /** diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java index 44360bc77d83f..3781c9c95f6a7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/AntiAffinityGroupPolicyHelper.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -41,11 +42,11 @@ public AntiAffinityGroupPolicyHelper(PulsarService pulsar, this.channel = channel; } - public void filter( - Map brokers, String bundle) { - LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, bundle, - brokers.keySet(), - channel.getOwnershipEntrySet(), brokerToFailureDomainMap); + public CompletableFuture> filterAsync(Map brokers, + String bundle) { + return LoadManagerShared.filterAntiAffinityGroupOwnedBrokersAsync(pulsar, bundle, + brokers.keySet(), channel.getOwnershipEntrySet(), brokerToFailureDomainMap) + .thenApply(__ -> brokers); } public boolean hasAntiAffinityGroupPolicy(String bundle) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java index 67dc702cc0c9f..468552db541ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/policies/IsolationPoliciesHelper.java @@ -18,10 +18,9 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.policies; -import io.netty.util.concurrent.FastThreadLocal; -import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; @@ -38,18 +37,9 @@ public IsolationPoliciesHelper(SimpleResourceAllocationPolicies policies) { this.policies = policies; } - private static final FastThreadLocal> localBrokerCandidateCache = new FastThreadLocal<>() { - @Override - protected Set initialValue() { - return new HashSet<>(); - } - }; - - public Set applyIsolationPolicies(Map availableBrokers, - ServiceUnitId serviceUnit) { - Set brokerCandidateCache = localBrokerCandidateCache.get(); - brokerCandidateCache.clear(); - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + public CompletableFuture> applyIsolationPoliciesAsync(Map availableBrokers, + ServiceUnitId serviceUnit) { + return LoadManagerShared.applyNamespacePoliciesAsync(serviceUnit, policies, availableBrokers.keySet(), new LoadManagerShared.BrokerTopicLoadingPredicate() { @Override public boolean isEnablePersistentTopics(String brokerUrl) { @@ -63,7 +53,6 @@ public boolean isEnableNonPersistentTopics(String brokerUrl) { return lookupData != null && lookupData.nonPersistentTopicsEnabled(); } }); - return brokerCandidateCache; } public boolean hasIsolationPolicy(NamespaceName namespaceName) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java index 07d521a28afa7..7bb16bac124e9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedder.java @@ -47,7 +47,6 @@ import lombok.experimental.Accessors; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.extensions.LoadManagerContext; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannel; @@ -720,8 +719,10 @@ private boolean isTransferable(LoadManagerContext context, Map candidates = new HashMap<>(availableBrokers); for (var filter : brokerFilterPipeline) { try { - filter.filter(candidates, namespaceBundle, context); - } catch (BrokerFilterException e) { + filter.filter(candidates, namespaceBundle, context) + .get(context.brokerConfiguration().getMetadataStoreOperationTimeoutSeconds(), + TimeUnit.SECONDS); + } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Failed to filter brokers with filter: {}", filter.getClass().getName(), e); return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index 33f346adbe0c5..5f2e4b1f25d8b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -181,6 +181,104 @@ public static void applyNamespacePolicies(final ServiceUnitId serviceUnit, } } + public static CompletableFuture> applyNamespacePoliciesAsync( + final ServiceUnitId serviceUnit, final SimpleResourceAllocationPolicies policies, + final Set availableBrokers, final BrokerTopicLoadingPredicate brokerTopicLoadingPredicate) { + NamespaceName namespace = serviceUnit.getNamespaceObject(); + return policies.areIsolationPoliciesPresentAsync(namespace).thenApply(isIsolationPoliciesPresent -> { + final Set brokerCandidateCache = new HashSet<>(); + Set primariesCache = localPrimariesCache.get(); + primariesCache.clear(); + + Set secondaryCache = localSecondaryCache.get(); + secondaryCache.clear(); + boolean isNonPersistentTopic = (serviceUnit instanceof NamespaceBundle) + ? ((NamespaceBundle) serviceUnit).hasNonPersistentTopic() : false; + if (isIsolationPoliciesPresent) { + if (LOG.isDebugEnabled()) { + LOG.debug("Isolation Policies Present for namespace - [{}]", namespace.toString()); + } + } + for (final String broker : availableBrokers) { + final String brokerUrlString = String.format("http://%s", broker); + URL brokerUrl; + try { + brokerUrl = new URL(brokerUrlString); + } catch (MalformedURLException e) { + LOG.error("Unable to parse brokerUrl from ResourceUnitId", e); + continue; + } + // todo: in future check if the resource unit has resources to take the namespace + if (isIsolationPoliciesPresent) { + // note: serviceUnitID is namespace name and ResourceID is brokerName + if (policies.isPrimaryBroker(namespace, brokerUrl.getHost())) { + primariesCache.add(broker); + if (LOG.isDebugEnabled()) { + LOG.debug("Added Primary Broker - [{}] as possible Candidates for" + + " namespace - [{}] with policies", brokerUrl.getHost(), namespace.toString()); + } + } else if (policies.isSecondaryBroker(namespace, brokerUrl.getHost())) { + secondaryCache.add(broker); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Added Shared Broker - [{}] as possible " + + "Candidates for namespace - [{}] with policies", + brokerUrl.getHost(), namespace.toString()); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping Broker - [{}] not primary broker and not shared" + + " for namespace - [{}] ", brokerUrl.getHost(), namespace.toString()); + } + + } + } else { + // non-persistent topic can be assigned to only those brokers that enabled for non-persistent topic + if (isNonPersistentTopic + && !brokerTopicLoadingPredicate.isEnableNonPersistentTopics(brokerUrlString)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Filter broker- [{}] because it doesn't support non-persistent namespace - [{}]", + brokerUrl.getHost(), namespace.toString()); + } + } else if (!isNonPersistentTopic + && !brokerTopicLoadingPredicate.isEnablePersistentTopics(brokerUrlString)) { + // persistent topic can be assigned to only brokers that enabled for persistent-topic + if (LOG.isDebugEnabled()) { + LOG.debug("Filter broker- [{}] because broker only supports non-persistent " + + "namespace - [{}]", brokerUrl.getHost(), namespace.toString()); + } + } else if (policies.isSharedBroker(brokerUrl.getHost())) { + secondaryCache.add(broker); + if (LOG.isDebugEnabled()) { + LOG.debug("Added Shared Broker - [{}] as possible Candidates for namespace - [{}]", + brokerUrl.getHost(), namespace.toString()); + } + } + } + } + if (isIsolationPoliciesPresent) { + brokerCandidateCache.addAll(primariesCache); + if (policies.shouldFailoverToSecondaries(namespace, primariesCache.size())) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Not enough of primaries [{}] available for namespace - [{}], " + + "adding shared [{}] as possible candidate owners", + primariesCache.size(), namespace.toString(), secondaryCache.size()); + } + brokerCandidateCache.addAll(secondaryCache); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug( + "Policies not present for namespace - [{}] so only " + + "considering shared [{}] brokers for possible owner", + namespace.toString(), secondaryCache.size()); + } + brokerCandidateCache.addAll(secondaryCache); + } + return brokerCandidateCache; + }); + } /** * Using the given bundles, populate the namespace to bundle range map. * @@ -408,6 +506,22 @@ public static void filterAntiAffinityGroupOwnedBrokers( } } + public static CompletableFuture filterAntiAffinityGroupOwnedBrokersAsync( + final PulsarService pulsar, final String assignedBundleName, + final Set candidates, + Set> bundleOwnershipData, + Map brokerToDomainMap + ) { + if (candidates.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + final String namespaceName = getNamespaceNameFromBundleName(assignedBundleName); + return getAntiAffinityNamespaceOwnedBrokers(pulsar, namespaceName, bundleOwnershipData) + .thenAccept(brokerToAntiAffinityNamespaceCount -> + filterAntiAffinityGroupOwnedBrokers(pulsar, candidates, + brokerToDomainMap, brokerToAntiAffinityNamespaceCount)); + } + /** * It computes least number of namespace owned by any of the domain and then it filters out all the domains that own * namespaces more than this count. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java index 4a1577a4e28b6..25fbc152b1163 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/SimpleResourceAllocationPolicies.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.loadbalance.LoadReport; import org.apache.pulsar.broker.loadbalance.ResourceUnit; @@ -53,6 +54,20 @@ private Optional getIsolationPolicies(String cluster } } + private CompletableFuture> getIsolationPoliciesAsync(String clusterName) { + return this.pulsar.getPulsarResources().getNamespaceResources() + .getIsolationPolicies().getIsolationDataPoliciesAsync(clusterName); + } + + public CompletableFuture areIsolationPoliciesPresentAsync(NamespaceName namespace) { + return getIsolationPoliciesAsync(pulsar.getConfiguration().getClusterName()) + .thenApply(policies -> { + return policies.filter(isolationPolicies -> + isolationPolicies.getPolicyByNamespace(namespace) != null) + .isPresent(); + }); + } + public boolean areIsolationPoliciesPresent(NamespaceName namespace) { try { Optional policies = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java index 3469dbe5a5499..8a73a5fa50006 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/AntiAffinityNamespaceGroupExtensionTest.java @@ -131,7 +131,7 @@ public void testAntiAffinityGroupPolicyFilter() var expected = new HashMap<>(brokers); var actual = antiAffinityGroupPolicyFilter.filter( - brokers, namespaceBundle, context); + brokers, namespaceBundle, context).get(); assertEquals(actual, expected); doReturn(antiAffinityEnabledNameSpace + "/" + bundle).when(namespaceBundle).toString(); @@ -142,7 +142,7 @@ public void testAntiAffinityGroupPolicyFilter() .get(5, TimeUnit.SECONDS).get(); expected.remove(srcBroker); actual = antiAffinityGroupPolicyFilter.filter( - brokers, namespaceBundle, context); + brokers, namespaceBundle, context).get(); assertEquals(actual, expected); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 0b51f27928bb7..533e6178d7944 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -106,6 +106,7 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.testcontainers.shaded.org.awaitility.Awaitility; @@ -267,11 +268,11 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { brokers.remove(pulsar1.getLookupServiceAddress()); - return brokers; + return CompletableFuture.completedFuture(brokers); } })).when(primaryLoadManager).getBrokerFilterPipeline(); @@ -288,11 +289,11 @@ public void testFilterHasException() throws Exception { doReturn(List.of(new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - brokers.clear(); - throw new BrokerFilterException("Test"); + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + brokers.remove(brokers.keySet().iterator().next()); + return FutureUtil.failedFuture(new BrokerFilterException("Test")); } })).when(primaryLoadManager).getBrokerFilterPipeline(); @@ -531,19 +532,18 @@ public void testMoreThenOneFilter() throws Exception { String lookupServiceAddress1 = pulsar1.getLookupServiceAddress(); doReturn(List.of(new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { brokers.remove(lookupServiceAddress1); - return brokers; + return CompletableFuture.completedFuture(brokers); } },new MockBrokerFilter() { @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - brokers.clear(); - throw new BrokerFilterException("Test"); + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return FutureUtil.failedFuture(new BrokerFilterException("Test")); } })).when(primaryLoadManager).getBrokerFilterPipeline(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java index c2c534f72e9db..b8d5d98fd4c08 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerIsolationPoliciesFilterTest.java @@ -31,6 +31,9 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; import org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl; @@ -67,7 +70,7 @@ public class BrokerIsolationPoliciesFilterTest { */ @Test public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBrokers() - throws IllegalAccessException, BrokerFilterException { + throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); @@ -83,18 +86,18 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker Map result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1")); // b. available-brokers: broker2, broker3 => result: broker2 result = filter.filter(new HashMap<>(Map.of( "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker2")); // c. available-brokers: broker3 => result: NULL result = filter.filter(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); // 2. Namespace: primary=broker1, secondary=broker2, shared=broker3, min_limit = 2 @@ -104,24 +107,24 @@ public void testFilterWithNamespaceIsolationPoliciesForPrimaryAndSecondaryBroker result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2")); // b. available-brokers: broker2, broker3 => result: broker2 result = filter.filter(new HashMap<>(Map.of( "broker2", getLookupData(), - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertEquals(result.keySet(), Set.of("broker2")); // c. available-brokers: broker3 => result: NULL result = filter.filter(new HashMap<>(Map.of( - "broker3", getLookupData())), namespaceName, getContext()); + "broker3", getLookupData())), namespaceName, getContext()).get(); assertTrue(result.isEmpty()); } @Test public void testFilterWithPersistentOrNonPersistentDisabled() - throws IllegalAccessException, BrokerFilterException { + throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { var namespace = "my-tenant/my-ns"; NamespaceName namespaceName = NamespaceName.get(namespace); NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); @@ -129,7 +132,8 @@ public void testFilterWithPersistentOrNonPersistentDisabled() doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); var policies = mock(SimpleResourceAllocationPolicies.class); - doReturn(false).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(false)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); doReturn(true).when(policies).isSharedBroker(any()); IsolationPoliciesHelper isolationPoliciesHelper = new IsolationPoliciesHelper(policies); @@ -140,14 +144,14 @@ public void testFilterWithPersistentOrNonPersistentDisabled() Map result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); + "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(true, false), "broker2", getLookupData(true, false), - "broker3", getLookupData())), namespaceBundle, getContext()); + "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker3")); doReturn(false).when(namespaceBundle).hasNonPersistentTopic(); @@ -155,13 +159,13 @@ public void testFilterWithPersistentOrNonPersistentDisabled() result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); + "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker1", "broker2", "broker3")); result = filter.filter(new HashMap<>(Map.of( "broker1", getLookupData(false, true), "broker2", getLookupData(), - "broker3", getLookupData())), namespaceBundle, getContext()); + "broker3", getLookupData())), namespaceBundle, getContext()).get(); assertEquals(result.keySet(), Set.of("broker2", "broker3")); } @@ -172,7 +176,8 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, Set shared, int min_limit) { reset(policies); - doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(true)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); doReturn(false).when(policies).isPrimaryBroker(eq(namespaceName), any()); doReturn(false).when(policies).isSecondaryBroker(eq(namespaceName), any()); doReturn(false).when(policies).isSharedBroker(any()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java index 4aef87cf63aa8..6b35bf71a0317 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerLoadManagerClassFilterTest.java @@ -27,6 +27,7 @@ import org.testng.annotations.Test; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; /** @@ -35,7 +36,7 @@ public class BrokerLoadManagerClassFilterTest extends BrokerFilterTestBase { @Test - public void test() throws BrokerFilterException { + public void test() throws BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); context.brokerConfiguration().setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); BrokerLoadManagerClassFilter filter = new BrokerLoadManagerClassFilter(); @@ -48,14 +49,14 @@ public void test() throws BrokerFilterException { "broker5", getLookupData("3.0.0", null) ); - Map result = filter.filter(new HashMap<>(originalBrokers), null, context); + Map result = filter.filter(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker1", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()), "broker2", getLookupData("3.0.0", ExtensibleLoadManagerImpl.class.getName()) )); context.brokerConfiguration().setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); - result = filter.filter(new HashMap<>(originalBrokers), null, context); + result = filter.filter(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker3", getLookupData("3.0.0", ModularLoadManagerImpl.class.getName()), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java index da13a9526a881..a6214304bd201 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerMaxTopicCountFilterTest.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.testng.Assert.assertEquals; @@ -38,7 +39,7 @@ public class BrokerMaxTopicCountFilterTest extends BrokerFilterTestBase { @Test - public void test() throws IllegalAccessException, BrokerFilterException { + public void test() throws IllegalAccessException, BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); LoadDataStore store = context.brokerLoadDataStore(); BrokerLoadData maxTopicLoadData = new BrokerLoadData(); @@ -58,7 +59,8 @@ public void test() throws IllegalAccessException, BrokerFilterException { "broker3", getLookupData(), "broker4", getLookupData() ); - Map result = filter.filter(new HashMap<>(originalBrokers), null, context); + Map result = + filter.filter(new HashMap<>(originalBrokers), null, context).get(); assertEquals(result, Map.of( "broker2", getLookupData(), "broker4", getLookupData() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java index cafd8f0ea7a4c..4a1a45439384e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/filter/BrokerVersionFilterTest.java @@ -21,9 +21,11 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.BrokerFilterBadVersionException; import org.apache.pulsar.broker.loadbalance.BrokerFilterException; @@ -39,14 +41,14 @@ public class BrokerVersionFilterTest extends BrokerFilterTestBase { @Test - public void testFilterEmptyBrokerList() throws BrokerFilterException { + public void testFilterEmptyBrokerList() throws BrokerFilterException, ExecutionException, InterruptedException { BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()); + Map result = brokerVersionFilter.filter(new HashMap<>(), null, getContext()).get(); assertTrue(result.isEmpty()); } @Test - public void testDisabledFilter() throws BrokerFilterException { + public void testDisabledFilter() throws BrokerFilterException, ExecutionException, InterruptedException { LoadManagerContext context = getContext(); ServiceConfiguration configuration = new ServiceConfiguration(); configuration.setPreferLaterVersions(false); @@ -58,12 +60,12 @@ public void testDisabledFilter() throws BrokerFilterException { ); Map brokers = new HashMap<>(originalBrokers); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - Map result = brokerVersionFilter.filter(brokers, null, context); + Map result = brokerVersionFilter.filter(brokers, null, context).get(); assertEquals(result, originalBrokers); } @Test - public void testFilter() throws BrokerFilterException { + public void testFilter() throws BrokerFilterException, ExecutionException, InterruptedException { Map originalBrokers = Map.of( "localhost:6650", getLookupData("2.10.0"), "localhost:6651", getLookupData("2.10.1"), @@ -72,7 +74,7 @@ public void testFilter() throws BrokerFilterException { ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); Map result = brokerVersionFilter.filter( - new HashMap<>(originalBrokers), null, getContext()); + new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6651", getLookupData("2.10.1"), "localhost:6652", getLookupData("2.10.1"), @@ -85,7 +87,7 @@ public void testFilter() throws BrokerFilterException { "localhost:6652", getLookupData("2.10.1"), "localhost:6653", getLookupData("2.10.1") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6652", getLookupData("2.10.1"), @@ -99,19 +101,24 @@ public void testFilter() throws BrokerFilterException { "localhost:6653", getLookupData("2.10.2-SNAPSHOT") ); - result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + result = brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); assertEquals(result, Map.of( "localhost:6653", getLookupData("2.10.2-SNAPSHOT") )); } - @Test(expectedExceptions = BrokerFilterBadVersionException.class) - public void testInvalidVersionString() throws BrokerFilterException { + @Test + public void testInvalidVersionString() { Map originalBrokers = Map.of( "localhost:6650", getLookupData("xxx") ); BrokerVersionFilter brokerVersionFilter = new BrokerVersionFilter(); - brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()); + try { + brokerVersionFilter.filter(new HashMap<>(originalBrokers), null, getContext()).get(); + fail(); + } catch (Exception ex) { + assertEquals(ex.getCause().getClass(), BrokerFilterBadVersionException.class); + } } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java index af5890fcacbb2..3dd41a92e682c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/scheduler/TransferShedderTest.java @@ -634,7 +634,8 @@ public void testBundlesWithIsolationPolicies() { // test setLoadBalancerSheddingBundlesWithPoliciesEnabled=false; - doReturn(true).when(allocationPoliciesSpy).areIsolationPoliciesPresent(any()); + doReturn(CompletableFuture.completedFuture(true)) + .when(allocationPoliciesSpy).areIsolationPoliciesPresentAsync(any()); ctx.brokerConfiguration().setLoadBalancerTransferEnabled(true); ctx.brokerConfiguration().setLoadBalancerSheddingBundlesWithPoliciesEnabled(false); res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); @@ -679,12 +680,14 @@ private void setIsolationPolicies(SimpleResourceAllocationPolicies policies, NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); doReturn(true).when(namespaceBundle).hasNonPersistentTopic(); doReturn(namespaceName).when(namespaceBundle).getNamespaceObject(); - doReturn(false).when(policies).areIsolationPoliciesPresent(any()); + doReturn(CompletableFuture.completedFuture(false)) + .when(policies).areIsolationPoliciesPresentAsync(any()); doReturn(false).when(policies).isPrimaryBroker(any(), any()); doReturn(false).when(policies).isSecondaryBroker(any(), any()); doReturn(true).when(policies).isSharedBroker(any()); - doReturn(true).when(policies).areIsolationPoliciesPresent(eq(namespaceName)); + doReturn(CompletableFuture.completedFuture(true)) + .when(policies).areIsolationPoliciesPresentAsync(eq(namespaceName)); primary.forEach(broker -> { doReturn(true).when(policies).isPrimaryBroker(eq(namespaceName), eq(broker)); @@ -723,8 +726,8 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { doAnswer(invocationOnMock -> { Map brokers = invocationOnMock.getArgument(0); brokers.clear(); - return brokers; - }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); + return CompletableFuture.completedFuture(brokers); + }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); var res = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); assertTrue(res.isEmpty()); @@ -737,11 +740,11 @@ public void testBundlesWithAntiAffinityGroup() throws MetadataStoreException { String bundle = invocationOnMock.getArgument(1, String.class); if (bundle.equalsIgnoreCase(bundleE1)) { - return brokers; + return CompletableFuture.completedFuture(brokers); } brokers.clear(); - return brokers; - }).when(antiAffinityGroupPolicyHelper).filter(any(), any()); + return CompletableFuture.completedFuture(brokers); + }).when(antiAffinityGroupPolicyHelper).filterAsync(any(), any()); var res2 = transferShedder.findBundlesForUnloading(ctx, Map.of(), Map.of()); var expected2 = new HashSet<>(); expected2.add(new UnloadDecision(new Unload("broker5", bundleE1, Optional.of("broker1")), @@ -761,10 +764,10 @@ public String name() { } @Override - public Map filter(Map brokers, - ServiceUnitId serviceUnit, - LoadManagerContext context) throws BrokerFilterException { - throw new BrokerFilterException("test"); + public CompletableFuture> filter(Map brokers, + ServiceUnitId serviceUnit, + LoadManagerContext context) { + return FutureUtil.failedFuture(new BrokerFilterException("test")); } }; filters.add(filter); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 86228d25a793f..51dc9d7dd72da 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -261,8 +261,7 @@ public void testStopBroker() throws Exception { assertNotEquals(broker1, broker); } - // TODO: This test is very flaky and it's disabled for now to unblock CI - @Test(timeOut = 40 * 1000, enabled = false) + @Test(timeOut = 40 * 1000) public void testAntiaffinityPolicy() throws PulsarAdminException { final String namespaceAntiAffinityGroup = "my-anti-affinity-filter"; final String antiAffinityEnabledNameSpace = DEFAULT_TENANT + "/my-ns-filter" + nsSuffix; From b5a29e7bb8395831d1bf7d08a45502eefc32c3c7 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 5 Jul 2023 09:29:03 +0800 Subject: [PATCH 190/494] [fix][broker] Revert "Skip loading broker interceptor when disableBrokerInterceptors is true #20422" (#20710) --- .../pulsar/broker/ServiceConfiguration.java | 2 +- .../broker/intercept/BrokerInterceptors.java | 5 ---- .../intercept/BrokerInterceptorTest.java | 25 ------------------- 3 files changed, 1 insertion(+), 31 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 8c85b9f28ab7e..2c48310f96482 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -1353,7 +1353,7 @@ The delayed message index time step(in seconds) in per bucket snapshot segment, @FieldContext( category = CATEGORY_SERVER, - doc = "Enable or disable the broker interceptor" + doc = "Enable or disable the broker interceptor, which is only used for testing for now" ) private boolean disableBrokerInterceptors = true; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java index 4ffd8732db94e..cef3f0eb609a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/intercept/BrokerInterceptors.java @@ -59,11 +59,6 @@ public BrokerInterceptors(Map intercep * @return the collection of broker event interceptor */ public static BrokerInterceptor load(ServiceConfiguration conf) throws IOException { - if (conf.isDisableBrokerInterceptors()) { - log.info("Skip loading the broker interceptors when disableBrokerInterceptors is true"); - return null; - } - BrokerInterceptorDefinitions definitions = BrokerInterceptorUtils.searchForInterceptors(conf.getBrokerInterceptorsDirectory(), conf.getNarExtractionDirectory()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java index c612104f8bf1b..d1cf91635f992 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/intercept/BrokerInterceptorTest.java @@ -20,12 +20,9 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; - import java.io.IOException; import java.util.HashMap; import java.util.HashSet; @@ -39,7 +36,6 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; -import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -311,25 +307,4 @@ public void requestInterceptorFailedTest() { } } - @Test - public void testLoadWhenDisableBrokerInterceptorsIsTrue() throws IOException { - ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); - serviceConfiguration.setDisableBrokerInterceptors(true); - BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration); - assertNull(brokerInterceptor); - - verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors(); - verify(serviceConfiguration, times(0)).getBrokerInterceptorsDirectory(); - } - - @Test - public void testLoadWhenDisableBrokerInterceptorsIsFalse() throws IOException { - ServiceConfiguration serviceConfiguration = spy(ServiceConfiguration.class); - serviceConfiguration.setDisableBrokerInterceptors(false); - BrokerInterceptor brokerInterceptor = BrokerInterceptors.load(serviceConfiguration); - assertNull(brokerInterceptor); - - verify(serviceConfiguration, times(1)).isDisableBrokerInterceptors(); - verify(serviceConfiguration, times(1)).getBrokerInterceptorsDirectory(); - } } From 35b449b6baf036da6caa8beb1d54e58aca6c654f Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 5 Jul 2023 13:42:59 +0800 Subject: [PATCH 191/494] Revert "[fix][broker] Fix NPE when reset Replicator's cursor by position. (#20597)" This reverts commit aa565c21c8103be8edbafaffa90029f90bce37da. --- .../admin/impl/PersistentTopicsBase.java | 42 +++++++------------ .../PersistentMessageExpiryMonitor.java | 16 +++---- .../persistent/PersistentReplicator.java | 2 +- .../persistent/PersistentSubscription.java | 2 +- .../service/PersistentMessageFinderTest.java | 20 +++------ .../pulsar/broker/service/ReplicatorTest.java | 30 ------------- 6 files changed, 28 insertions(+), 84 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 03e1a1b27475a..81f5e3c1f323b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -4145,43 +4145,31 @@ private CompletableFuture internalExpireMessagesNonPartitionedTopicByPosit return; } try { - PersistentSubscription sub = null; - PersistentReplicator repl = null; - - if (subName.startsWith(topic.getReplicatorPrefix())) { - String remoteCluster = PersistentReplicator.getRemoteCluster(subName); - repl = (PersistentReplicator) - topic.getPersistentReplicator(remoteCluster); - if (repl == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - "Replicator not found")); - return; - } - } else { - sub = topic.getSubscription(subName); - if (sub == null) { - asyncResponse.resume(new RestException(Status.NOT_FOUND, - getSubNotFoundErrorMessage(topicName.toString(), subName))); - return; - } + PersistentSubscription sub = topic.getSubscription(subName); + if (sub == null) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + getSubNotFoundErrorMessage(topicName.toString(), subName))); + return; } - CompletableFuture batchSizeFuture = new CompletableFuture<>(); getEntryBatchSize(batchSizeFuture, topic, messageId, batchIndex); - - PersistentReplicator finalRepl = repl; - PersistentSubscription finalSub = sub; - batchSizeFuture.thenAccept(bi -> { PositionImpl position = calculatePositionAckSet(isExcluded, bi, batchIndex, messageId); boolean issued; try { if (subName.startsWith(topic.getReplicatorPrefix())) { - issued = finalRepl.expireMessages(position); + String remoteCluster = PersistentReplicator.getRemoteCluster(subName); + PersistentReplicator repl = (PersistentReplicator) + topic.getPersistentReplicator(remoteCluster); + if (repl == null) { + asyncResponse.resume(new RestException(Status.NOT_FOUND, + "Replicator not found")); + return; + } + issued = repl.expireMessages(position); } else { - issued = finalSub.expireMessages(position); + issued = sub.expireMessages(position); } - if (issued) { log.info("[{}] Message expire started up to {} on {} {}", clientAppId(), position, topicName, subName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index 3acd683f9f42b..bddcb1b334df1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -22,7 +22,6 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.LongAdder; -import javax.annotation.Nullable; import org.apache.bookkeeper.mledger.AsyncCallbacks.FindEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -44,7 +43,6 @@ public class PersistentMessageExpiryMonitor implements FindEntryCallback { private final ManagedCursor cursor; private final String subName; - private final PersistentTopic topic; private final String topicName; private final Rate msgExpired; private final LongAdder totalMsgExpired; @@ -59,10 +57,9 @@ public class PersistentMessageExpiryMonitor implements FindEntryCallback { expirationCheckInProgressUpdater = AtomicIntegerFieldUpdater .newUpdater(PersistentMessageExpiryMonitor.class, "expirationCheckInProgress"); - public PersistentMessageExpiryMonitor(PersistentTopic topic, String subscriptionName, ManagedCursor cursor, - @Nullable PersistentSubscription subscription) { - this.topic = topic; - this.topicName = topic.getName(); + public PersistentMessageExpiryMonitor(String topicName, String subscriptionName, ManagedCursor cursor, + PersistentSubscription subscription) { + this.topicName = topicName; this.cursor = cursor; this.subName = subscriptionName; this.subscription = subscription; @@ -101,12 +98,11 @@ public boolean expireMessages(int messageTTLInSeconds) { public boolean expireMessages(Position messagePosition) { // If it's beyond last position of this topic, do nothing. - PositionImpl topicLastPosition = (PositionImpl) this.topic.getLastPosition(); - if (topicLastPosition.compareTo((PositionImpl) messagePosition) < 0) { + if (((PositionImpl) subscription.getTopic().getLastPosition()).compareTo((PositionImpl) messagePosition) < 0) { if (log.isDebugEnabled()) { log.debug("[{}][{}] Ignore expire-message scheduled task, given position {} is beyond " - + "current topic's last position {}", topicName, subName, messagePosition, - topicLastPosition); + + "current topic's last position {}", topicName, subName, messagePosition, + subscription.getTopic().getLastPosition()); } return false; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index d468cb0643f72..ccf70eecec35c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -116,7 +116,7 @@ public PersistentReplicator(String localCluster, PersistentTopic localTopic, Man brokerService, replicationClient); this.topic = localTopic; this.cursor = cursor; - this.expiryMonitor = new PersistentMessageExpiryMonitor(localTopic, + this.expiryMonitor = new PersistentMessageExpiryMonitor(localTopicName, Codec.decode(cursor.getName()), cursor, null); HAVE_PENDING_READ_UPDATER.set(this, FALSE); PENDING_MESSAGES_UPDATER.set(this, 0); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 192e5ed3e006a..090e8e253776d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -152,7 +152,7 @@ public PersistentSubscription(PersistentTopic topic, String subscriptionName, Ma this.topicName = topic.getName(); this.subName = subscriptionName; this.fullName = MoreObjects.toStringHelper(this).add("topic", topicName).add("name", subName).toString(); - this.expiryMonitor = new PersistentMessageExpiryMonitor(topic, subscriptionName, cursor, this); + this.expiryMonitor = new PersistentMessageExpiryMonitor(topicName, subscriptionName, cursor, this); this.setReplicated(replicated); this.subscriptionProperties = MapUtils.isEmpty(subscriptionProperties) ? Collections.emptyMap() : Collections.unmodifiableMap(subscriptionProperties); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java index f0e2e6eafcdfb..e77fd07c6ef8b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentMessageFinderTest.java @@ -62,7 +62,6 @@ import org.apache.pulsar.broker.service.persistent.PersistentMessageExpiryMonitor; import org.apache.pulsar.broker.service.persistent.PersistentMessageFinder; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; -import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ResetCursorData; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; @@ -231,11 +230,7 @@ public void findEntryFailed(ManagedLedgerException exception, Optional }); assertTrue(ex.get()); - PersistentTopic mock = mock(PersistentTopic.class); - when(mock.getName()).thenReturn("topicname"); - when(mock.getLastPosition()).thenReturn(PositionImpl.EARLIEST); - - PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor("topicname", c1.getName(), c1, null); monitor.findEntryFailed(new ManagedLedgerException.ConcurrentFindCursorPositionException("failed"), Optional.empty(), null); Field field = monitor.getClass().getDeclaredField("expirationCheckInProgress"); @@ -412,11 +407,7 @@ void testMessageExpiryWithTimestampNonRecoverableException() throws Exception { bkc.deleteLedger(ledgers.get(1).getLedgerId()); bkc.deleteLedger(ledgers.get(2).getLedgerId()); - PersistentTopic mock = mock(PersistentTopic.class); - when(mock.getName()).thenReturn("topicname"); - when(mock.getLastPosition()).thenReturn(PositionImpl.EARLIEST); - - PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor(mock, c1.getName(), c1, null); + PersistentMessageExpiryMonitor monitor = new PersistentMessageExpiryMonitor("topicname", c1.getName(), c1, null); Position previousMarkDelete = null; for (int i = 0; i < totalEntries; i++) { monitor.expireMessages(1); @@ -453,16 +444,15 @@ void testMessageExpiryWithPosition() throws Exception { ManagedCursorImpl cursor = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName); PersistentSubscription subscription = mock(PersistentSubscription.class); - PersistentTopic topic = mock(PersistentTopic.class); + Topic topic = mock(Topic.class); when(subscription.getTopic()).thenReturn(topic); - when(topic.getName()).thenReturn("topicname"); for (int i = 0; i < totalEntries; i++) { positions.add(ledger.addEntry(createMessageWrittenToLedger("msg" + i))); } when(topic.getLastPosition()).thenReturn(positions.get(positions.size() - 1)); - PersistentMessageExpiryMonitor monitor = spy(new PersistentMessageExpiryMonitor(topic, + PersistentMessageExpiryMonitor monitor = spy(new PersistentMessageExpiryMonitor("topicname", cursor.getName(), cursor, subscription)); assertEquals(cursor.getMarkDeletedPosition(), PositionImpl.get(positions.get(0).getLedgerId(), -1)); boolean issued; @@ -501,7 +491,7 @@ void testMessageExpiryWithPosition() throws Exception { clearInvocations(monitor); ManagedCursorImpl mockCursor = mock(ManagedCursorImpl.class); - PersistentMessageExpiryMonitor mockMonitor = spy(new PersistentMessageExpiryMonitor(topic, + PersistentMessageExpiryMonitor mockMonitor = spy(new PersistentMessageExpiryMonitor("topicname", cursor.getName(), mockCursor, subscription)); // Not calling findEntryComplete to clear expirationCheckInProgress condition, so following call to // expire message shouldn't issue. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java index f710c8541d1b5..176eab0e94b3d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorTest.java @@ -728,36 +728,6 @@ public void testReplicatorClearBacklog() throws Exception { assertEquals(status.getReplicationBacklog(), 0); } - - @Test(timeOut = 30000) - public void testResetReplicatorSubscriptionPosition() throws Exception { - final TopicName dest = TopicName - .get(BrokerTestUtil.newUniqueName("persistent://pulsar/ns/resetReplicatorSubscription")); - - @Cleanup - MessageProducer producer1 = new MessageProducer(url1, dest); - - // Produce from cluster1 and consume from the rest - for (int i = 0; i < 10; i++) { - producer1.produce(2); - } - - PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString()).get(); - - PersistentReplicator replicator = (PersistentReplicator) spy( - topic.getReplicators().get(topic.getReplicators().keys().get(0))); - - MessageId id = topic.getLastMessageId().get(); - admin1.topics().expireMessages(dest.getPartitionedTopicName(), - replicator.getCursor().getName(), - id,false); - - replicator.updateRates(); - - ReplicatorStats status = replicator.getStats(); - assertEquals(status.getReplicationBacklog(), 0); - } - @Test(timeOut = 30000) public void testResetCursorNotFail() throws Exception { From 894192fb6542e504be43034a3c33e90f9c6e528a Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 6 Jul 2023 08:04:29 -0500 Subject: [PATCH 192/494] [fix][ws] Remove unnecessary ping/pong implementation (#20733) (cherry picked from commit 33044f0e0f38374256d5b89edff6a23d08f4759d) --- .../apache/pulsar/broker/PulsarService.java | 7 -- .../proxy/server/ProxyServiceStarter.java | 7 -- .../proxy/server/ProxyServiceStarterTest.java | 12 ---- .../pulsar/websocket/PingPongHandler.java | 50 --------------- .../websocket/WebSocketPingPongServlet.java | 44 ------------- .../service/WebSocketServiceStarter.java | 4 -- ...dlerTest.java => PingPongSupportTest.java} | 64 +++++++++++++++++-- 7 files changed, 58 insertions(+), 130 deletions(-) delete mode 100644 pulsar-websocket/src/main/java/org/apache/pulsar/websocket/PingPongHandler.java delete mode 100644 pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketPingPongServlet.java rename pulsar-websocket/src/test/java/org/apache/pulsar/websocket/{PingPongHandlerTest.java => PingPongSupportTest.java} (67%) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 62d4634fa2d62..cf8ab33fd7ee4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -173,7 +173,6 @@ import org.apache.pulsar.transaction.coordinator.TransactionMetadataStoreProvider; import org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStoreProvider; import org.apache.pulsar.websocket.WebSocketConsumerServlet; -import org.apache.pulsar.websocket.WebSocketPingPongServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; import org.apache.pulsar.websocket.WebSocketReaderServlet; import org.apache.pulsar.websocket.WebSocketService; @@ -1072,12 +1071,6 @@ private void addWebSocketServiceHandler(WebService webService, new ServletHolder(readerWebSocketServlet), true, attributeMap); webService.addServlet(WebSocketReaderServlet.SERVLET_PATH_V2, new ServletHolder(readerWebSocketServlet), true, attributeMap); - - final WebSocketServlet pingPongWebSocketServlet = new WebSocketPingPongServlet(webSocketService); - webService.addServlet(WebSocketPingPongServlet.SERVLET_PATH, - new ServletHolder(pingPongWebSocketServlet), true, attributeMap); - webService.addServlet(WebSocketPingPongServlet.SERVLET_PATH_V2, - new ServletHolder(pingPongWebSocketServlet), true, attributeMap); } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index beee9f1a4f763..46a5efef66853 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -51,7 +51,6 @@ import org.apache.pulsar.common.util.ShutdownUtil; import org.apache.pulsar.proxy.stats.ProxyStats; import org.apache.pulsar.websocket.WebSocketConsumerServlet; -import org.apache.pulsar.websocket.WebSocketPingPongServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; import org.apache.pulsar.websocket.WebSocketReaderServlet; import org.apache.pulsar.websocket.WebSocketService; @@ -315,12 +314,6 @@ public static void addWebServerHandlers(WebServer server, new ServletHolder(readerWebSocketServlet)); server.addServlet(WebSocketReaderServlet.SERVLET_PATH_V2, new ServletHolder(readerWebSocketServlet)); - - final WebSocketServlet pingPongWebSocketServlet = new WebSocketPingPongServlet(webSocketService); - server.addServlet(WebSocketPingPongServlet.SERVLET_PATH, - new ServletHolder(pingPongWebSocketServlet)); - server.addServlet(WebSocketPingPongServlet.SERVLET_PATH_V2, - new ServletHolder(pingPongWebSocketServlet)); } } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java index 4dcfc17096448..def58be6df372 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java @@ -76,18 +76,6 @@ private String computeWsBasePath() { return String.format("ws://localhost:%d/ws", serviceStarter.getServer().getListenPortHTTP().get()); } - @Test - public void testEnableWebSocketServer() throws Exception { - HttpClient httpClient = new HttpClient(); - WebSocketClient webSocketClient = new WebSocketClient(httpClient); - webSocketClient.start(); - MyWebSocket myWebSocket = new MyWebSocket(); - String webSocketUri = computeWsBasePath() + "/pingpong"; - Future sessionFuture = webSocketClient.connect(myWebSocket, URI.create(webSocketUri)); - System.out.println("uri" + webSocketUri); - sessionFuture.get().getRemote().sendPing(ByteBuffer.wrap("ping".getBytes())); - assertTrue(myWebSocket.getResponse().contains("ping")); - } @Test public void testProducer() throws Exception { diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/PingPongHandler.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/PingPongHandler.java deleted file mode 100644 index 870630abc8889..0000000000000 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/PingPongHandler.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.websocket; - -import java.io.IOException; -import java.nio.ByteBuffer; -import org.eclipse.jetty.util.BufferUtil; -import org.eclipse.jetty.websocket.api.WebSocketAdapter; -import org.eclipse.jetty.websocket.api.WebSocketPingPongListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PingPongHandler extends WebSocketAdapter implements WebSocketPingPongListener { - - private static final Logger log = LoggerFactory.getLogger(PingPongHandler.class); - - @Override - public void onWebSocketPing(ByteBuffer payload) { - try { - if (log.isDebugEnabled()) { - log.debug("PING: {}", BufferUtil.toDetailString(payload)); - } - getRemote().sendPong(payload); - } catch (IOException e) { - log.warn("Failed to send pong: {}", e.getMessage()); - } - } - - @Override - public void onWebSocketPong(ByteBuffer payload) { - - } - -} \ No newline at end of file diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketPingPongServlet.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketPingPongServlet.java deleted file mode 100644 index cc2d79ee541ba..0000000000000 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/WebSocketPingPongServlet.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.websocket; - -import org.eclipse.jetty.websocket.servlet.WebSocketServlet; -import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; - -public class WebSocketPingPongServlet extends WebSocketServlet { - private static final long serialVersionUID = 1L; - - public static final String SERVLET_PATH = "/ws/pingpong"; - public static final String SERVLET_PATH_V2 = "/ws/v2/pingpong"; - - private final transient WebSocketService service; - - public WebSocketPingPongServlet(WebSocketService service) { - this.service = service; - } - - @Override - public void configure(WebSocketServletFactory factory) { - factory.getPolicy().setMaxTextMessageSize(service.getConfig().getWebSocketMaxTextFrameSize()); - if (service.getConfig().getWebSocketSessionIdleTimeoutMillis() > 0) { - factory.getPolicy().setIdleTimeout(service.getConfig().getWebSocketSessionIdleTimeoutMillis()); - } - factory.setCreator((request, response) -> new PingPongHandler()); - } -} \ No newline at end of file diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java index fbcecc0642e34..6231ef1a2aa41 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/WebSocketServiceStarter.java @@ -29,7 +29,6 @@ import org.apache.pulsar.common.util.CmdGenerateDocs; import org.apache.pulsar.common.util.ShutdownUtil; import org.apache.pulsar.websocket.WebSocketConsumerServlet; -import org.apache.pulsar.websocket.WebSocketPingPongServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; import org.apache.pulsar.websocket.WebSocketReaderServlet; import org.apache.pulsar.websocket.WebSocketService; @@ -91,7 +90,6 @@ public static void start(ProxyServer proxyServer, WebSocketService service) thro proxyServer.addWebSocketServlet(WebSocketProducerServlet.SERVLET_PATH, new WebSocketProducerServlet(service)); proxyServer.addWebSocketServlet(WebSocketConsumerServlet.SERVLET_PATH, new WebSocketConsumerServlet(service)); proxyServer.addWebSocketServlet(WebSocketReaderServlet.SERVLET_PATH, new WebSocketReaderServlet(service)); - proxyServer.addWebSocketServlet(WebSocketPingPongServlet.SERVLET_PATH, new WebSocketPingPongServlet(service)); proxyServer.addWebSocketServlet(WebSocketProducerServlet.SERVLET_PATH_V2, new WebSocketProducerServlet(service)); @@ -99,8 +97,6 @@ public static void start(ProxyServer proxyServer, WebSocketService service) thro new WebSocketConsumerServlet(service)); proxyServer.addWebSocketServlet(WebSocketReaderServlet.SERVLET_PATH_V2, new WebSocketReaderServlet(service)); - proxyServer.addWebSocketServlet(WebSocketPingPongServlet.SERVLET_PATH_V2, - new WebSocketPingPongServlet(service)); proxyServer.addRestResource(ADMIN_PATH_V1, ATTRIBUTE_PROXY_SERVICE_NAME, service, WebSocketProxyStatsV1.class); proxyServer.addRestResource(ADMIN_PATH_V2, ATTRIBUTE_PROXY_SERVICE_NAME, service, WebSocketProxyStatsV2.class); diff --git a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongHandlerTest.java b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongSupportTest.java similarity index 67% rename from pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongHandlerTest.java rename to pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongSupportTest.java index 662009f1aab1a..8119c2f1f8131 100644 --- a/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongHandlerTest.java +++ b/pulsar-websocket/src/test/java/org/apache/pulsar/websocket/PingPongSupportTest.java @@ -18,13 +18,16 @@ */ package org.apache.pulsar.websocket; +import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.Future; +import javax.servlet.http.HttpServletRequest; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.web.WebExecutorThreadPool; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.server.Server; @@ -40,11 +43,18 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.testng.Assert.assertTrue; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; +import org.eclipse.jetty.websocket.servlet.WebSocketServlet; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class PingPongHandlerTest { +/** + * Test to ensure {@link AbstractWebSocketHandler} has ping/pong support + */ +public class PingPongSupportTest { private static Server server; @@ -67,9 +77,9 @@ public static void setup() throws Exception { when(config.getWebSocketMaxTextFrameSize()).thenReturn(1048576); when(config.getWebSocketSessionIdleTimeoutMillis()).thenReturn(300000); - ServletHolder servletHolder = new ServletHolder("ws-events", new WebSocketPingPongServlet(service)); + ServletHolder servletHolder = new ServletHolder("ws-events", new GenericWebSocketServlet(service)); ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); - context.setContextPath(WebSocketPingPongServlet.SERVLET_PATH); + context.setContextPath("/ws"); context.addServlet(servletHolder, "/*"); server.setHandler(context); try { @@ -87,18 +97,60 @@ public static void tearDown() throws Exception { executor.stop(); } - @Test - public void testPingPong() throws Exception { + /** + * We test these different endpoints because they are parsed in the AbstractWebSocketHandler. Technically, we are + * not testing these implementations, but the ping/pong support is guaranteed as part of the framework. + */ + @DataProvider(name = "endpoint") + public static Object[][] cacheEnable() { + return new Object[][] { { "producer" }, { "consumer" }, { "reader" } }; + } + + @Test(dataProvider = "endpoint") + public void testPingPong(String endpoint) throws Exception { HttpClient httpClient = new HttpClient(); WebSocketClient webSocketClient = new WebSocketClient(httpClient); webSocketClient.start(); MyWebSocket myWebSocket = new MyWebSocket(); - String webSocketUri = "ws://localhost:8080/ws/pingpong"; + String webSocketUri = "ws://localhost:8080/ws/v2/" + endpoint + "/persistent/my-property/my-ns/my-topic"; Future sessionFuture = webSocketClient.connect(myWebSocket, URI.create(webSocketUri)); sessionFuture.get().getRemote().sendPing(ByteBuffer.wrap("test".getBytes())); assertTrue(myWebSocket.getResponse().contains("test")); } + public static class GenericWebSocketHandler extends AbstractWebSocketHandler { + + public GenericWebSocketHandler(WebSocketService service, HttpServletRequest request, ServletUpgradeResponse response) { + super(service, request, response); + } + + @Override + protected Boolean isAuthorized(String authRole, AuthenticationDataSource authenticationData) throws Exception { + return true; + } + + @Override + public void close() throws IOException { + + } + } + + public static class GenericWebSocketServlet extends WebSocketServlet { + + private static final long serialVersionUID = 1L; + private final WebSocketService service; + + public GenericWebSocketServlet(WebSocketService service) { + this.service = service; + } + + @Override + public void configure(WebSocketServletFactory factory) { + factory.setCreator((request, response) -> + new GenericWebSocketHandler(service, request.getHttpServletRequest(), response)); + } + } + @WebSocket public static class MyWebSocket extends WebSocketAdapter implements WebSocketPingPongListener { From 6e2f6683da80ac0ba2f3b2a6078408fdb87c477a Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Thu, 6 Jul 2023 20:45:26 -0500 Subject: [PATCH 193/494] [fix] Ignore openIDTokenIssuerTrustCertsFilePath conf when blank (#20745) (cherry picked from commit 4586852f30902b5247d907ce4cc9360152db1e21) --- .../authentication/oidc/AuthenticationProviderOpenID.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java index 2078666a08dd9..1462b8e293f79 100644 --- a/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java +++ b/pulsar-broker-auth-oidc/src/main/java/org/apache/pulsar/broker/authentication/oidc/AuthenticationProviderOpenID.java @@ -52,6 +52,7 @@ import java.util.concurrent.CompletableFuture; import javax.naming.AuthenticationException; import javax.net.ssl.SSLSession; +import org.apache.commons.lang.StringUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.AuthenticationProvider; @@ -163,7 +164,9 @@ public void initialize(ServiceConfiguration config) throws IOException { int readTimeout = getConfigValueAsInt(config, HTTP_READ_TIMEOUT_MILLIS, HTTP_READ_TIMEOUT_MILLIS_DEFAULT); String trustCertsFilePath = getConfigValueAsString(config, ISSUER_TRUST_CERTS_FILE_PATH, null); SslContext sslContext = null; - if (trustCertsFilePath != null) { + // When config is in the conf file but is empty, it defaults to the empty string, which is not meaningful and + // should be ignored. + if (StringUtils.isNotBlank(trustCertsFilePath)) { // Use default settings for everything but the trust store. sslContext = SslContextBuilder.forClient() .trustManager(new File(trustCertsFilePath)) From f3bb89d4a685a36ffaa1dffa962b90387328fbca Mon Sep 17 00:00:00 2001 From: Jiwe Guo Date: Mon, 10 Jul 2023 17:28:17 +0800 Subject: [PATCH 194/494] Revert Add listener interface for namespace service #20406 --- .../channel/ServiceUnitStateChannelImpl.java | 1 - .../NamespaceBundleSplitListener.java | 29 ---------------- .../broker/namespace/NamespaceService.java | 28 ---------------- .../ExtensibleLoadManagerImplTest.java | 19 ----------- .../namespace/NamespaceCreateBundlesTest.java | 33 ------------------- 5 files changed, 110 deletions(-) delete mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 717ff484fe772..4eb25848fda94 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -930,7 +930,6 @@ protected void splitServiceUnitOnceAndRetry(NamespaceService namespaceService, double splitBundleTime = TimeUnit.NANOSECONDS.toMillis((System.nanoTime() - startTime)); log.info("Successfully split {} parent namespace-bundle to {} in {} ms", parentBundle, childBundles, splitBundleTime); - namespaceService.onNamespaceBundleSplit(parentBundle); completionFuture.complete(null); }) .exceptionally(ex -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java deleted file mode 100644 index a3312f5689e38..0000000000000 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceBundleSplitListener.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.namespace; - -import java.util.function.Predicate; -import org.apache.pulsar.common.naming.NamespaceBundle; - -/** - * Listener for NamespaceBundle split. - */ -public interface NamespaceBundleSplitListener extends Predicate { - void onSplit(NamespaceBundle bundle); -} \ No newline at end of file diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 9be8d4938e3e3..141e48515b29b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -139,10 +139,6 @@ public class NamespaceService implements AutoCloseable { private final ConcurrentOpenHashMap namespaceClients; private final List bundleOwnershipListeners; - - private final List bundleSplitListeners; - - private final RedirectManager redirectManager; @@ -171,7 +167,6 @@ public NamespaceService(PulsarService pulsar) { this.namespaceClients = ConcurrentOpenHashMap.newBuilder().build(); this.bundleOwnershipListeners = new CopyOnWriteArrayList<>(); - this.bundleSplitListeners = new CopyOnWriteArrayList<>(); this.localBrokerDataCache = pulsar.getLocalMetadataStore().getMetadataCache(LocalBrokerData.class); this.redirectManager = new RedirectManager(pulsar); } @@ -1018,7 +1013,6 @@ void splitAndOwnBundleOnceAndRetry(NamespaceBundle bundle, // affect the split operation which is already safely completed r.forEach(this::unloadNamespaceBundle); } - onNamespaceBundleSplit(bundle); }) .exceptionally(e -> { String msg1 = format( @@ -1261,19 +1255,6 @@ public void onNamespaceBundleUnload(NamespaceBundle bundle) { } } } - - public void onNamespaceBundleSplit(NamespaceBundle bundle) { - for (NamespaceBundleSplitListener bundleSplitListener : bundleSplitListeners) { - try { - if (bundleSplitListener.test(bundle)) { - bundleSplitListener.onSplit(bundle); - } - } catch (Throwable t) { - LOG.error("Call bundle {} split listener {} error", bundle, bundleSplitListener, t); - } - } - } - public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener... listeners) { Objects.requireNonNull(listeners); for (NamespaceBundleOwnershipListener listener : listeners) { @@ -1284,15 +1265,6 @@ public void addNamespaceBundleOwnershipListener(NamespaceBundleOwnershipListener getOwnedServiceUnits().forEach(bundle -> notifyNamespaceBundleOwnershipListener(bundle, listeners)); } - public void addNamespaceBundleSplitListener(NamespaceBundleSplitListener... listeners) { - Objects.requireNonNull(listeners); - for (NamespaceBundleSplitListener listener : listeners) { - if (listener != null) { - bundleSplitListeners.add(listener); - } - } - } - private void notifyNamespaceBundleOwnershipListener(NamespaceBundle bundle, NamespaceBundleOwnershipListener... listeners) { if (listeners != null) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 533e6178d7944..cff92ab27c654 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -88,7 +88,6 @@ import org.apache.pulsar.broker.lookup.LookupResult; import org.apache.pulsar.broker.namespace.LookupOptions; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; -import org.apache.pulsar.broker.namespace.NamespaceBundleSplitListener; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; @@ -406,23 +405,6 @@ public void testSplitBundleAdminAPI() throws Exception { String firstBundle = bundleRanges.get(0) + "_" + bundleRanges.get(1); - AtomicInteger splitCount = new AtomicInteger(0); - NamespaceBundleSplitListener namespaceBundleSplitListener = new NamespaceBundleSplitListener() { - @Override - public void onSplit(NamespaceBundle bundle) { - splitCount.incrementAndGet(); - } - - @Override - public boolean test(NamespaceBundle namespaceBundle) { - return namespaceBundle - .toString() - .equals(String.format(namespace + "/0x%08x_0x%08x", bundleRanges.get(0), bundleRanges.get(1))); - } - }; - pulsar1.getNamespaceService().addNamespaceBundleSplitListener(namespaceBundleSplitListener); - pulsar2.getNamespaceService().addNamespaceBundleSplitListener(namespaceBundleSplitListener); - long mid = bundleRanges.get(0) + (bundleRanges.get(1) - bundleRanges.get(0)) / 2; admin.namespaces().splitNamespaceBundle(namespace, firstBundle, true, null); @@ -435,7 +417,6 @@ public boolean test(NamespaceBundle namespaceBundle) { assertTrue(bundlesData.getBoundaries().contains(lowBundle)); assertTrue(bundlesData.getBoundaries().contains(midBundle)); assertTrue(bundlesData.getBoundaries().contains(highBundle)); - assertEquals(splitCount.get(), 1); // Test split bundle with invalid bundle range. try { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java index 73cfaf1b0d96b..43d37466918ce 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceCreateBundlesTest.java @@ -20,21 +20,15 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; - -import lombok.Cleanup; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; -import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.policies.data.BookieAffinityGroupData; import org.apache.pulsar.common.policies.data.Policies; -import org.awaitility.Awaitility; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -87,31 +81,4 @@ public void testSplitBundleUpdatesLocalPoliciesWithoutOverwriting() throws Excep assertNotNull(admin.namespaces().getBookieAffinityGroup(namespaceName)); producer.close(); } - - @Test - public void testBundleSplitListener() throws Exception { - String namespaceName = "prop/" + UUID.randomUUID().toString(); - String topicName = "persistent://" + namespaceName + "/my-topic5"; - admin.namespaces().createNamespace(namespaceName); - @Cleanup - Producer producer = pulsarClient.newProducer().topic(topicName).sendTimeout(1, - TimeUnit.SECONDS).create(); - producer.send(new byte[1]); - String bundleRange = admin.lookups().getBundleRange(topicName); - AtomicBoolean isTriggered = new AtomicBoolean(false); - pulsar.getNamespaceService().addNamespaceBundleSplitListener(new NamespaceBundleSplitListener() { - @Override - public void onSplit(NamespaceBundle bundle) { - assertEquals(bundleRange, bundle.getBundleRange()); - isTriggered.set(true); - } - - @Override - public boolean test(NamespaceBundle namespaceBundle) { - return true; - } - }); - admin.namespaces().splitNamespaceBundle(namespaceName, bundleRange, false, null); - Awaitility.await().untilAsserted(() -> assertTrue(isTriggered.get())); - } } From 0cd4bed49f49b0f5906968e0b6785a74d1f3e409 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 20 Jun 2023 19:36:57 +0300 Subject: [PATCH 195/494] [fix][sec] Upgrade snappy-java to address multiple CVEs (#20604) (cherry picked from commit 62a99ed1c9c602ca35607e2141d3772734d4c5b7) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 4b8534a165035..3d07f168cb18e 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -480,7 +480,7 @@ The Apache Software License, Version 2.0 - org.apache.zookeeper-zookeeper-jute-3.8.1.jar - org.apache.zookeeper-zookeeper-prometheus-metrics-3.8.1.jar * Snappy Java - - org.xerial.snappy-snappy-java-1.1.8.4.jar + - org.xerial.snappy-snappy-java-1.1.10.1.jar * Google HTTP Client - com.google.http-client-google-http-client-gson-1.41.0.jar - com.google.http-client-google-http-client-1.41.0.jar diff --git a/pom.xml b/pom.xml index b63af0581805f..84a5bbaad1a34 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ flexible messaging model and an intuitive client API. 3.8.1 1.5.0 1.10.0 - 1.1.8.4 + 1.1.10.1 4.1.12.1 5.1.0 4.1.93.Final diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 0623c0a306f17..d56b87a3fb7f8 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -454,7 +454,7 @@ The Apache Software License, Version 2.0 * JSON Simple - json-simple-1.1.1.jar * Snappy - - snappy-java-1.1.8.4.jar + - snappy-java-1.1.10.1.jar * Jackson - jackson-module-parameter-names-2.14.2.jar * Java Assist From d7e863748abbd7d6408aafa924cffb7766f3ab0f Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Thu, 13 Jul 2023 10:30:32 +0800 Subject: [PATCH 196/494] Release 3.0.1 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 132 files changed, 134 insertions(+), 134 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index cf31bd6c64a01..5c4c62b1b8d8a 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index a2eb2d1437082..cb2a36d6b5685 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 25b8a32a4b755..9e2f46435ff58 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 5ccb3122dd163..9590ece2e0629 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 1a02938fc531a..7bc3e34b13fec 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,12 +31,12 @@ org.apache.pulsar buildtools - 3.0.1-SNAPSHOT + 3.0.1 jar Pulsar Build Tools - 2023-04-12T05:18:48Z + 2023-07-13T02:30:03Z 1.8 1.8 3.1.0 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 7f8f7b33c2e38..8803f2d6c90ef 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index d0c37bd25210b..cae5539821e88 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index d9b72f01eaf3f..5cf02f2f19fc3 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 1171f8acaeb8f..05988001502b4 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index a8855daefcecd..8bdc59c562770 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/docker/pom.xml b/docker/pom.xml index 6e6b27c7d7533..6d553e59f221a 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.1-SNAPSHOT + 3.0.1 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index d4b9e35aff315..b27d0b081873b 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.1-SNAPSHOT + 3.0.1 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 7ec474aca9d89..8761e57dbc340 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.1-SNAPSHOT + 3.0.1 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 441f3b11dc5bf..ee2a64b1f320a 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 6934d1a8432af..a4659122188b2 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1-SNAPSHOT + 3.0.1 .. diff --git a/pom.xml b/pom.xml index 84a5bbaad1a34..4cad368b4b8dc 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 3.0.1-SNAPSHOT + 3.0.1 Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -92,7 +92,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2023-04-12T05:18:48Z + 2023-07-13T02:30:03Z true 4.1.12.1 5.1.0 - 4.1.93.Final + 4.1.94.Final 0.0.21.Final 9.4.51.v20230217 2.5.2 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d56b87a3fb7f8..e70bcee56ecae 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -231,21 +231,21 @@ The Apache Software License, Version 2.0 - commons-compress-1.21.jar - commons-lang3-3.11.jar * Netty - - netty-buffer-4.1.93.Final.jar - - netty-codec-4.1.93.Final.jar - - netty-codec-dns-4.1.93.Final.jar - - netty-codec-http-4.1.93.Final.jar - - netty-codec-haproxy-4.1.93.Final.jar - - netty-codec-socks-4.1.93.Final.jar - - netty-handler-proxy-4.1.93.Final.jar - - netty-common-4.1.93.Final.jar - - netty-handler-4.1.93.Final.jar + - netty-buffer-4.1.94.Final.jar + - netty-codec-4.1.94.Final.jar + - netty-codec-dns-4.1.94.Final.jar + - netty-codec-http-4.1.94.Final.jar + - netty-codec-haproxy-4.1.94.Final.jar + - netty-codec-socks-4.1.94.Final.jar + - netty-handler-proxy-4.1.94.Final.jar + - netty-common-4.1.94.Final.jar + - netty-handler-4.1.94.Final.jar - netty-reactive-streams-2.0.6.jar - - netty-resolver-4.1.93.Final.jar - - netty-resolver-dns-4.1.93.Final.jar - - netty-resolver-dns-classes-macos-4.1.93.Final.jar - - netty-resolver-dns-native-macos-4.1.93.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.93.Final-osx-x86_64.jar + - netty-resolver-4.1.94.Final.jar + - netty-resolver-dns-4.1.94.Final.jar + - netty-resolver-dns-classes-macos-4.1.94.Final.jar + - netty-resolver-dns-native-macos-4.1.94.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.94.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.61.Final.jar - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar @@ -253,12 +253,12 @@ The Apache Software License, Version 2.0 - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar - netty-tcnative-classes-2.0.61.Final.jar - - netty-transport-4.1.93.Final.jar - - netty-transport-classes-epoll-4.1.93.Final.jar - - netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.93.Final.jar - - netty-transport-native-unix-common-4.1.93.Final-linux-x86_64.jar - - netty-codec-http2-4.1.93.Final.jar + - netty-transport-4.1.94.Final.jar + - netty-transport-classes-epoll-4.1.94.Final.jar + - netty-transport-native-epoll-4.1.94.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.94.Final.jar + - netty-transport-native-unix-common-4.1.94.Final-linux-x86_64.jar + - netty-codec-http2-4.1.94.Final.jar - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar From a302772e391c6162e9e9554d217e2fb88915d047 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Sat, 19 Aug 2023 10:03:10 -0500 Subject: [PATCH 244/494] [improve][proxy] Support disabling metrics endpoint (#21031) (cherry picked from commit d06cda6cd8a58b8a7e0678183f05c08059ddb9b2) --- conf/proxy.conf | 2 + .../proxy/server/ProxyConfiguration.java | 6 +++ .../proxy/server/ProxyServiceStarter.java | 18 +++++---- .../apache/pulsar/proxy/server/WebServer.java | 24 +++++++----- .../apache/pulsar/proxy/stats/ProxyStats.java | 39 ++++++++++++++++++- 5 files changed, 72 insertions(+), 17 deletions(-) diff --git a/conf/proxy.conf b/conf/proxy.conf index a33db8bbf7a30..44d950824e394 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -369,6 +369,8 @@ zooKeeperCacheExpirySeconds=-1 ### --- Metrics --- ### +# Whether to enable the proxy's /metrics, /proxy-stats, and /status.html http endpoints +enableProxyStatsEndpoints=true # Whether the '/metrics' endpoint requires authentication. Defaults to true authenticateMetricsEndpoint=true # Enable cache metrics data, default value is false diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 3ecd670cbbf7a..a4cb7926bebf1 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -371,6 +371,12 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private int authenticationRefreshCheckSeconds = 60; + @FieldContext( + category = CATEGORY_HTTP, + doc = "Whether to enable the proxy's /metrics, /proxy-stats, and /status.html http endpoints" + ) + private boolean enableProxyStatsEndpoints = true; + @FieldContext( category = CATEGORY_AUTHENTICATION, doc = "Whether the '/metrics' endpoint requires authentication. Defaults to true." diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index 46a5efef66853..d0774cee88301 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -255,15 +255,19 @@ public static void addWebServerHandlers(WebServer server, ProxyConfiguration config, ProxyService service, BrokerDiscoveryProvider discoveryProvider) throws Exception { - if (service != null) { - PrometheusMetricsServlet metricsServlet = service.getMetricsServlet(); - if (metricsServlet != null) { - server.addServlet("/metrics", new ServletHolder(metricsServlet), - Collections.emptyList(), config.isAuthenticateMetricsEndpoint()); + if (config.isEnableProxyStatsEndpoints()) { + server.addRestResource("/", VipStatus.ATTRIBUTE_STATUS_FILE_PATH, config.getStatusFilePath(), + VipStatus.class); + server.addRestResource("/proxy-stats", ProxyStats.ATTRIBUTE_PULSAR_PROXY_NAME, service, + ProxyStats.class); + if (service != null) { + PrometheusMetricsServlet metricsServlet = service.getMetricsServlet(); + if (metricsServlet != null) { + server.addServlet("/metrics", new ServletHolder(metricsServlet), + Collections.emptyList(), config.isAuthenticateMetricsEndpoint()); + } } } - server.addRestResource("/", VipStatus.ATTRIBUTE_STATUS_FILE_PATH, config.getStatusFilePath(), VipStatus.class); - server.addRestResource("/proxy-stats", ProxyStats.ATTRIBUTE_PULSAR_PROXY_NAME, service, ProxyStats.class); AdminProxyHandler adminProxyHandler = new AdminProxyHandler(config, discoveryProvider); ServletHolder servletHolder = new ServletHolder(adminProxyHandler); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java index 1ca8dc93ebf9e..edbcfe0847c4e 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java @@ -197,12 +197,20 @@ public void addServlet(String basePath, ServletHolder servletHolder, List> attributes, boolean requireAuthentication) { + addServlet(basePath, servletHolder, attributes, requireAuthentication, true); + } + + private void addServlet(String basePath, ServletHolder servletHolder, + List> attributes, boolean requireAuthentication, boolean checkForExistingPaths) { popularServletParams(servletHolder, config); - Optional existingPath = servletPaths.stream().filter(p -> p.startsWith(basePath)).findFirst(); - if (existingPath.isPresent()) { - throw new IllegalArgumentException( - String.format("Cannot add servlet at %s, path %s already exists", basePath, existingPath.get())); + if (checkForExistingPaths) { + Optional existingPath = servletPaths.stream().filter(p -> p.startsWith(basePath)).findFirst(); + if (existingPath.isPresent()) { + throw new IllegalArgumentException( + String.format("Cannot add servlet at %s, path %s already exists", basePath, + existingPath.get())); + } } servletPaths.add(basePath); @@ -237,11 +245,9 @@ public void addRestResource(String basePath, String attribute, Object attributeV config.register(JsonMapperProvider.class); ServletHolder servletHolder = new ServletHolder(new ServletContainer(config)); servletHolder.setAsyncSupported(true); - ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); - context.setContextPath(basePath); - context.addServlet(servletHolder, MATCH_ALL); - context.setAttribute(attribute, attributeValue); - handlers.add(context); + // This method has not historically checked for existing paths, so we don't check here either. The + // method call is added to reduce code duplication. + addServlet(basePath, servletHolder, Collections.singletonList(Pair.of(attribute, attributeValue)), true, false); } public int getExternalServicePort() { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java index 27e61c90e9e26..3c85b5f2b71cc 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.proxy.stats; +import static java.util.concurrent.TimeUnit.SECONDS; import io.netty.channel.Channel; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -27,7 +28,10 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -36,7 +40,12 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; +import org.apache.pulsar.broker.web.AuthenticationFilter; import org.apache.pulsar.proxy.server.ProxyService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @@ -45,12 +54,15 @@ @Produces(MediaType.APPLICATION_JSON) public class ProxyStats { + private static final Logger log = LoggerFactory.getLogger(ProxyStats.class); public static final String ATTRIBUTE_PULSAR_PROXY_NAME = "pulsar-proxy"; private ProxyService service; @Context protected ServletContext servletContext; + @Context + protected HttpServletRequest httpRequest; @GET @Path("/connections") @@ -58,6 +70,7 @@ public class ProxyStats { response = List.class, responseContainer = "List") @ApiResponses(value = { @ApiResponse(code = 503, message = "Proxy service is not initialized") }) public List metrics() { + throwIfNotSuperUser("metrics"); List stats = new ArrayList<>(); proxyService().getClientCnxs().forEach(cnx -> { if (cnx.getDirectProxyHandler() == null) { @@ -78,7 +91,7 @@ public List metrics() { @ApiResponses(value = { @ApiResponse(code = 412, message = "Proxy logging should be > 2 to capture topic stats"), @ApiResponse(code = 503, message = "Proxy service is not initialized") }) public Map topics() { - + throwIfNotSuperUser("topics"); Optional logLevel = proxyService().getConfiguration().getProxyLogLevel(); if (!logLevel.isPresent() || logLevel.get() < 2) { throw new RestException(Status.PRECONDITION_FAILED, "Proxy doesn't have logging level 2"); @@ -92,6 +105,7 @@ public Map topics() { notes = "It only changes log-level in memory, change it config file to persist the change") @ApiResponses(value = { @ApiResponse(code = 412, message = "Proxy log level can be [0-2]"), }) public void updateProxyLogLevel(@PathParam("logLevel") int logLevel) { + throwIfNotSuperUser("updateProxyLogLevel"); if (logLevel < 0 || logLevel > 2) { throw new RestException(Status.PRECONDITION_FAILED, "Proxy log level can be only [0-2]"); } @@ -102,6 +116,7 @@ public void updateProxyLogLevel(@PathParam("logLevel") int logLevel) { @Path("/logging") @ApiOperation(hidden = true, value = "Get proxy logging") public int getProxyLogLevel(@PathParam("logLevel") int logLevel) { + throwIfNotSuperUser("getProxyLogLevel"); return proxyService().getProxyLogLevel(); } @@ -114,4 +129,26 @@ protected ProxyService proxyService() { } return service; } + + private void throwIfNotSuperUser(String action) { + if (proxyService().getConfiguration().isAuthorizationEnabled()) { + AuthenticationParameters authParams = AuthenticationParameters.builder() + .clientRole((String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName)) + .clientAuthenticationDataSource((AuthenticationDataSource) + httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName)) + .build(); + try { + if (authParams.getClientRole() == null + || !proxyService().getAuthorizationService().isSuperUser(authParams).get(30, SECONDS)) { + log.error("Client with role [{}] is not authorized to {}", authParams.getClientRole(), action); + throw new org.apache.pulsar.common.util.RestException(Status.UNAUTHORIZED, + "Client is not authorized to perform operation"); + } + } catch (ExecutionException | TimeoutException | InterruptedException e) { + log.warn("Time-out {} sec while checking the role {} is a super user role ", 30, + authParams.getClientRole()); + throw new org.apache.pulsar.common.util.RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + } } From bc1019fa8ed37b8a4c8bb01e3662c6c015e1bc27 Mon Sep 17 00:00:00 2001 From: Yiheng Cao <65160922+Crispy-fried-chicken@users.noreply.github.com> Date: Fri, 25 Aug 2023 00:41:32 +0800 Subject: [PATCH 245/494] [fix][broker] Use MessageDigest.isEqual when comparing digests (#21061) (cherry picked from commit c05954e66ff33098aeb848f4bde51613ace7e47e) --- .../pulsar/broker/authentication/SaslRoleTokenSigner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java index c979566e485ce..82f760e14b7d6 100644 --- a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java +++ b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java @@ -76,7 +76,7 @@ public String verifyAndExtract(String signedStr) throws AuthenticationException String originalSignature = signedStr.substring(index + SIGNATURE.length()); String rawValue = signedStr.substring(0, index); String currentSignature = computeSignature(rawValue); - if (!originalSignature.equals(currentSignature)) { + if (!MessageDigest.isEqual(originalSignature.getBytes(), currentSignature.getBytes())){ throw new AuthenticationException("Invalid signature"); } return rawValue; From 5f999251ed44bc31caf0cedc6511a790cbfe83ba Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 25 Aug 2023 19:21:21 +0800 Subject: [PATCH 246/494] [fix] [bk] Correctct the bookie info after ZK client is reconnected (#21035) Motivation: After [PIP-118: reconnect broker when ZooKeeper session expires](https://github.com/apache/pulsar/pull/13341), the Broker will not shut down after losing the connection of the local metadata store in the default configuration. However, before the ZK client is reconnected, the events of BK online and offline are lost, resulting in incorrect BK info in the memory. You can reproduce the issue by the test `BkEnsemblesChaosTest. testBookieInfoIsCorrectEvenIfLostNotificationDueToZKClientReconnect`(90% probability of reproduce of the issue, run it again if the issue does not occur) Modifications: Refresh BK info in memory after the ZK client is reconnected. (cherry picked from commit db20035bba9eb21a6b3e4e6752ab716b9df35d80) --- .../broker/service/BkEnsemblesChaosTest.java | 71 ++++++ ...econnectZKClientPulsarServiceBaseTest.java | 215 ++++++++++++++++++ .../pulsar/metadata/api/MetadataCache.java | 5 + .../bookkeeper/PulsarRegistrationClient.java | 23 +- .../metadata/impl/AbstractMetadataStore.java | 8 + 5 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java new file mode 100644 index 0000000000000..d49489d8a84b0 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Producer; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class BkEnsemblesChaosTest extends CanReconnectZKClientPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testBookieInfoIsCorrectEvenIfLostNotificationDueToZKClientReconnect() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + final byte[] msgValue = "test".getBytes(); + admin.topics().createNonPartitionedTopic(topicName); + // Ensure broker works. + Producer producer1 = client.newProducer().topic(topicName).create(); + producer1.send(msgValue); + producer1.close(); + admin.topics().unload(topicName); + + // Restart some bookies, which triggers the ZK node of Bookie deleted and created. + // And make the local metadata store reconnect to lose some notification of the ZK node change. + for (int i = 0; i < numberOfBookies - 1; i++){ + bkEnsemble.stopBK(i); + } + makeLocalMetadataStoreKeepReconnect(); + for (int i = 0; i < numberOfBookies - 1; i++){ + bkEnsemble.startBK(i); + } + // Sleep 100ms to lose the notifications of ZK node create. + Thread.sleep(100); + stopLocalMetadataStoreAlwaysReconnect(); + + // Ensure broker still works. + admin.topics().unload(topicName); + Producer producer2 = client.newProducer().topic(topicName).create(); + producer2.send(msgValue); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java new file mode 100644 index 0000000000000..bc6df685ffcd7 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Sets; +import io.netty.channel.Channel; +import java.net.URL; +import java.nio.channels.SelectionKey; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.apache.zookeeper.ClientCnxn; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.reflect.WhiteboxImpl; + +@Slf4j +public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetrySupport { + + protected final String defaultTenant = "public"; + protected final String defaultNamespace = defaultTenant + "/default"; + protected int numberOfBookies = 3; + protected final String clusterName = "r1"; + protected URL url; + protected URL urlTls; + protected ServiceConfiguration config = new ServiceConfiguration(); + protected ZookeeperServerTest brokerConfigZk; + protected LocalBookkeeperEnsemble bkEnsemble; + protected PulsarService pulsar; + protected BrokerService broker; + protected PulsarAdmin admin; + protected PulsarClient client; + protected ZooKeeper localZkOfBroker; + protected Object localMetaDataStoreClientCnx; + protected final AtomicBoolean LocalMetadataStoreInReconnectFinishSignal = new AtomicBoolean(); + protected void startZKAndBK() throws Exception { + // Start ZK. + brokerConfigZk = new ZookeeperServerTest(0); + brokerConfigZk.start(); + + // Start BK. + bkEnsemble = new LocalBookkeeperEnsemble(numberOfBookies, 0, () -> 0); + bkEnsemble.start(); + } + + protected void startBrokers() throws Exception { + // Start brokers. + setConfigDefaults(config, clusterName, bkEnsemble, brokerConfigZk); + pulsar = new PulsarService(config); + pulsar.start(); + broker = pulsar.getBrokerService(); + ZKMetadataStore zkMetadataStore = (ZKMetadataStore) pulsar.getLocalMetadataStore(); + localZkOfBroker = zkMetadataStore.getZkClient(); + ClientCnxn cnxn = WhiteboxImpl.getInternalState(localZkOfBroker, "cnxn"); + Object sendThread = WhiteboxImpl.getInternalState(cnxn, "sendThread"); + localMetaDataStoreClientCnx = WhiteboxImpl.getInternalState(sendThread, "clientCnxnSocket"); + + url = new URL(pulsar.getWebServiceAddress()); + urlTls = new URL(pulsar.getWebServiceAddressTls()); + admin = PulsarAdmin.builder().serviceHttpUrl(url.toString()).build(); + client = PulsarClient.builder().serviceUrl(url.toString()).build(); + } + + protected void makeLocalMetadataStoreKeepReconnect() throws Exception { + if (!LocalMetadataStoreInReconnectFinishSignal.compareAndSet(false, true)) { + throw new RuntimeException("Local metadata store is already keeping reconnect"); + } + if (localMetaDataStoreClientCnx.getClass().getSimpleName().equals("ClientCnxnSocketNIO")) { + makeLocalMetadataStoreKeepReconnectNIO(); + } else { + // ClientCnxnSocketNetty. + makeLocalMetadataStoreKeepReconnectNetty(); + } + } + + protected void makeLocalMetadataStoreKeepReconnectNIO() { + new Thread(() -> { + while (LocalMetadataStoreInReconnectFinishSignal.get()) { + try { + SelectionKey sockKey = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "sockKey"); + if (sockKey != null) { + sockKey.channel().close(); + } + // Prevents high cpu usage. + Thread.sleep(5); + } catch (Exception e) { + log.error("Try close the ZK connection of local metadata store failed: {}", e.toString()); + } + } + }).start(); + } + + protected void makeLocalMetadataStoreKeepReconnectNetty() { + new Thread(() -> { + while (LocalMetadataStoreInReconnectFinishSignal.get()) { + try { + Channel channel = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "channel"); + if (channel != null) { + channel.close(); + } + // Prevents high cpu usage. + Thread.sleep(5); + } catch (Exception e) { + log.error("Try close the ZK connection of local metadata store failed: {}", e.toString()); + } + } + }).start(); + } + + protected void stopLocalMetadataStoreAlwaysReconnect() { + LocalMetadataStoreInReconnectFinishSignal.set(false); + } + + protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { + admin.clusters().createCluster(clusterName, ClusterData.builder() + .serviceUrl(url.toString()) + .serviceUrlTls(urlTls.toString()) + .brokerServiceUrl(pulsar.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + + admin.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(clusterName))); + + admin.namespaces().createNamespace(defaultNamespace, Sets.newHashSet(clusterName)); + } + + @Override + protected void setup() throws Exception { + incrementSetupNumber(); + + log.info("--- Starting OneWayReplicatorTestBase::setup ---"); + + startZKAndBK(); + + startBrokers(); + + createDefaultTenantsAndClustersAndNamespace(); + + Thread.sleep(100); + log.info("--- OneWayReplicatorTestBase::setup completed ---"); + } + + private void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setWebServicePort(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bookkeeperEnsemble.getZookeeperPort()); + config.setConfigurationMetadataStoreUrl("zk:127.0.0.1:" + brokerConfigZk.getZookeeperPort() + "/foo"); + config.setBrokerDeleteInactiveTopicsEnabled(false); + config.setBrokerDeleteInactiveTopicsFrequencySeconds(60); + config.setBrokerShutdownTimeoutMs(0L); + config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); + config.setBacklogQuotaCheckIntervalInSeconds(5); + config.setDefaultNumberOfNamespaceBundles(1); + config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); + config.setEnableReplicatedSubscriptions(true); + config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + } + + @Override + protected void cleanup() throws Exception { + markCurrentSetupNumberCleaned(); + log.info("--- Shutting down ---"); + + stopLocalMetadataStoreAlwaysReconnect(); + + // Stop brokers. + client.close(); + admin.close(); + if (pulsar != null) { + pulsar.close(); + } + + // Stop ZK and BK. + bkEnsemble.stop(); + brokerConfigZk.stop(); + + // Reset configs. + config = new ServiceConfiguration(); + setConfigDefaults(config, clusterName, bkEnsemble, brokerConfigZk); + } +} diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java index 94da382b74dcf..6d558e709716d 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java @@ -148,6 +148,11 @@ public interface MetadataCache { */ void invalidate(String path); + /** + * Force the invalidation of all object in the metadata cache. + */ + void invalidateAll(); + /** * Invalidate and reload an object in the metadata cache. * diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java index 306b6398b5c50..be945d988fb88 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java @@ -51,11 +51,13 @@ import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.Notification; +import org.apache.pulsar.metadata.api.extended.SessionEvent; +import org.apache.pulsar.metadata.impl.AbstractMetadataStore; @Slf4j public class PulsarRegistrationClient implements RegistrationClient { - private final MetadataStore store; + private final AbstractMetadataStore store; private final String ledgersRootPath; // registration paths private final String bookieRegistrationPath; @@ -68,10 +70,11 @@ public class PulsarRegistrationClient implements RegistrationClient { private final Map> writableBookieInfo; private final Map> readOnlyBookieInfo; private final FutureUtil.Sequencer sequencer; + private SessionEvent lastMetadataSessionEvent; public PulsarRegistrationClient(MetadataStore store, String ledgersRootPath) { - this.store = store; + this.store = (AbstractMetadataStore) store; this.ledgersRootPath = ledgersRootPath; this.bookieServiceInfoMetadataCache = store.getMetadataCache(BookieServiceInfoSerde.INSTANCE); this.sequencer = Sequencer.create(); @@ -88,6 +91,7 @@ public PulsarRegistrationClient(MetadataStore store, .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-registration-client")); store.registerListener(this::updatedBookies); + this.store.registerSessionListener(this::refreshBookies); } @Override @@ -95,6 +99,21 @@ public void close() { executor.shutdownNow(); } + private void refreshBookies(SessionEvent sessionEvent) { + lastMetadataSessionEvent = sessionEvent; + if (!SessionEvent.Reconnected.equals(sessionEvent) && !SessionEvent.SessionReestablished.equals(sessionEvent)){ + return; + } + // Clean caches. + store.invalidateCaches(bookieRegistrationPath, bookieAllRegistrationPath, bookieReadonlyRegistrationPath); + bookieServiceInfoMetadataCache.invalidateAll(); + // Refresh caches of the listeners. + getReadOnlyBookies().thenAccept(bookies -> + readOnlyBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); + getWritableBookies().thenAccept(bookies -> + writableBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); + } + @Override public CompletableFuture>> getWritableBookies() { return getBookiesThenFreshCache(bookieRegistrationPath); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index 4cadf2397a7fa..cc148c2a3117a 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -23,6 +23,7 @@ import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.DefaultThreadFactory; import java.time.Instant; @@ -523,6 +524,13 @@ public void invalidateAll() { existsCache.synchronous().invalidateAll(); } + public void invalidateCaches(String...paths) { + LoadingCache> loadingCache = childrenCache.synchronous(); + for (String path : paths) { + loadingCache.invalidate(path); + } + } + /** * Run the task in the executor thread and fail the future if the executor is shutting down. */ From 111c14d16156fd7a364a53c1bedf23b34da9526c Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Tue, 22 Aug 2023 18:07:09 +0800 Subject: [PATCH 247/494] [fix][broker] Fix get topic policies as null during clean cache (#20763) (cherry picked from commit 3116abf30f7f4c8a2a5e608e1ca672e2bada3e2d) --- .../pulsar/compaction/TwoPhaseCompactor.java | 1 + .../pulsar/compaction/CompactorTest.java | 95 ++++++++++++++----- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java index 21ccb281f0515..d71227d94f59f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java @@ -283,6 +283,7 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map promise.complete(null); } }); + return; } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java index e86be6a4db816..5f83ef97d32ca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java @@ -19,13 +19,10 @@ package org.apache.pulsar.compaction; import static org.apache.pulsar.client.impl.RawReaderTest.extractKey; - import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import io.netty.buffer.ByteBuf; - import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; @@ -33,23 +30,30 @@ import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; - +import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawMessageImpl; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.Commands; +import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -65,7 +69,6 @@ public class CompactorTest extends MockedPulsarServiceBaseTest { protected Compactor compactor; - @BeforeMethod @Override public void setup() throws Exception { @@ -101,16 +104,17 @@ protected Compactor getCompactor() { return compactor; } - private List compactAndVerify(String topic, Map expected, boolean checkMetrics) throws Exception { + private List compactAndVerify(String topic, Map expected, boolean checkMetrics) + throws Exception { long compactedLedgerId = compact(topic); LedgerHandle ledger = bk.openLedger(compactedLedgerId, - Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, - Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); Assert.assertEquals(ledger.getLastAddConfirmed() + 1, // 0..lac - expected.size(), - "Should have as many entries as there is keys"); + expected.size(), + "Should have as many entries as there is keys"); List keys = new ArrayList<>(); Enumeration entries = ledger.readEntries(0, ledger.getLastAddConfirmed()); @@ -124,7 +128,7 @@ private List compactAndVerify(String topic, Map expected byte[] bytes = new byte[payload.readableBytes()]; payload.readBytes(bytes); Assert.assertEquals(bytes, expected.remove(key), - "Compacted version should match expected version"); + "Compacted version should match expected version"); m.close(); } if (checkMetrics) { @@ -148,17 +152,18 @@ public void testCompaction() throws Exception { final int numMessages = 1000; final int maxKeys = 10; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); Random r = new Random(0); for (int j = 0; j < numMessages; j++) { int keyIndex = r.nextInt(maxKeys); - String key = "key"+keyIndex; + String key = "key" + keyIndex; byte[] data = ("my-message-" + key + "-" + j).getBytes(); producer.newMessage() .key(key) @@ -173,10 +178,11 @@ public void testCompaction() throws Exception { public void testCompactAddCompact() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); @@ -210,10 +216,11 @@ public void testCompactAddCompact() throws Exception { public void testCompactedInOrder() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); producer.newMessage() .key("c") @@ -256,6 +263,50 @@ public void testPhaseOneLoopTimeConfiguration() { Assert.assertEquals(compactor.getPhaseOneLoopReadTimeoutInSeconds(), 60); } + @Test + public void testCompactedWithConcurrentSend() throws Exception { + String topic = "persistent://my-property/use/my-ns/testCompactedWithConcurrentSend"; + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + BookKeeper bk = pulsar.getBookKeeperClientFactory().create( + this.conf, null, null, Optional.empty(), null); + Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + + CompletableFuture future = CompletableFuture.runAsync(() -> { + for (int i = 0; i < 100; i++) { + try { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i).getBytes()).send(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + }); + + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + CompactedTopic compactedTopic = persistentTopic.getCompactedTopic(); + + Awaitility.await().untilAsserted(() -> { + long compactedLedgerId = compactor.compact(topic).join(); + Thread.sleep(300); + Optional compactedTopicContext = persistentTopic.getCompactedTopicContext(); + Assert.assertTrue(compactedTopicContext.isPresent()); + Assert.assertEquals(compactedTopicContext.get().ledger.getId(), compactedLedgerId); + }); + + Position lastCompactedPosition = compactedTopic.getCompactionHorizon().get(); + Entry lastCompactedEntry = compactedTopic.readLastEntryOfCompactedLedger().get(); + + Assert.assertTrue(PositionImpl.get(lastCompactedPosition.getLedgerId(), lastCompactedPosition.getEntryId()) + .compareTo(lastCompactedEntry.getLedgerId(), lastCompactedEntry.getEntryId()) >= 0); + + future.join(); + } + public ByteBuf extractPayload(RawMessage m) throws Exception { ByteBuf payloadAndMetadata = m.getHeadersAndPayload(); Commands.skipChecksumIfPresent(payloadAndMetadata); From ab9384b02f2c8b46a4c84121f3c709d8cbad2201 Mon Sep 17 00:00:00 2001 From: coderzc Date: Tue, 29 Aug 2023 09:42:21 +0800 Subject: [PATCH 248/494] Revert "[fix][broker] Fix get topic policies as null during clean cache (#20763)" This reverts commit 111c14d16156fd7a364a53c1bedf23b34da9526c. --- .../pulsar/compaction/TwoPhaseCompactor.java | 1 - .../pulsar/compaction/CompactorTest.java | 95 +++++-------------- 2 files changed, 22 insertions(+), 74 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java index d71227d94f59f..21ccb281f0515 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java @@ -283,7 +283,6 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map promise.complete(null); } }); - return; } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java index 5f83ef97d32ca..e86be6a4db816 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java @@ -19,10 +19,13 @@ package org.apache.pulsar.compaction; import static org.apache.pulsar.client.impl.RawReaderTest.extractKey; + import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; + import io.netty.buffer.ByteBuf; + import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; @@ -30,30 +33,23 @@ import java.util.Map; import java.util.Optional; import java.util.Random; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import lombok.Cleanup; + import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.mledger.Entry; -import org.apache.bookkeeper.mledger.Position; -import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawMessageImpl; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.protocol.Commands; -import org.awaitility.Awaitility; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -69,6 +65,7 @@ public class CompactorTest extends MockedPulsarServiceBaseTest { protected Compactor compactor; + @BeforeMethod @Override public void setup() throws Exception { @@ -104,17 +101,16 @@ protected Compactor getCompactor() { return compactor; } - private List compactAndVerify(String topic, Map expected, boolean checkMetrics) - throws Exception { + private List compactAndVerify(String topic, Map expected, boolean checkMetrics) throws Exception { long compactedLedgerId = compact(topic); LedgerHandle ledger = bk.openLedger(compactedLedgerId, - Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, - Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); Assert.assertEquals(ledger.getLastAddConfirmed() + 1, // 0..lac - expected.size(), - "Should have as many entries as there is keys"); + expected.size(), + "Should have as many entries as there is keys"); List keys = new ArrayList<>(); Enumeration entries = ledger.readEntries(0, ledger.getLastAddConfirmed()); @@ -128,7 +124,7 @@ private List compactAndVerify(String topic, Map expected byte[] bytes = new byte[payload.readableBytes()]; payload.readBytes(bytes); Assert.assertEquals(bytes, expected.remove(key), - "Compacted version should match expected version"); + "Compacted version should match expected version"); m.close(); } if (checkMetrics) { @@ -152,18 +148,17 @@ public void testCompaction() throws Exception { final int numMessages = 1000; final int maxKeys = 10; - @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); Random r = new Random(0); for (int j = 0; j < numMessages; j++) { int keyIndex = r.nextInt(maxKeys); - String key = "key" + keyIndex; + String key = "key"+keyIndex; byte[] data = ("my-message-" + key + "-" + j).getBytes(); producer.newMessage() .key(key) @@ -178,11 +173,10 @@ public void testCompaction() throws Exception { public void testCompactAddCompact() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; - @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); @@ -216,11 +210,10 @@ public void testCompactAddCompact() throws Exception { public void testCompactedInOrder() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; - @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); producer.newMessage() .key("c") @@ -263,50 +256,6 @@ public void testPhaseOneLoopTimeConfiguration() { Assert.assertEquals(compactor.getPhaseOneLoopReadTimeoutInSeconds(), 60); } - @Test - public void testCompactedWithConcurrentSend() throws Exception { - String topic = "persistent://my-property/use/my-ns/testCompactedWithConcurrentSend"; - - @Cleanup - Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); - - BookKeeper bk = pulsar.getBookKeeperClientFactory().create( - this.conf, null, null, Optional.empty(), null); - Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); - - CompletableFuture future = CompletableFuture.runAsync(() -> { - for (int i = 0; i < 100; i++) { - try { - producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i).getBytes()).send(); - } catch (PulsarClientException e) { - throw new RuntimeException(e); - } - } - }); - - PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); - CompactedTopic compactedTopic = persistentTopic.getCompactedTopic(); - - Awaitility.await().untilAsserted(() -> { - long compactedLedgerId = compactor.compact(topic).join(); - Thread.sleep(300); - Optional compactedTopicContext = persistentTopic.getCompactedTopicContext(); - Assert.assertTrue(compactedTopicContext.isPresent()); - Assert.assertEquals(compactedTopicContext.get().ledger.getId(), compactedLedgerId); - }); - - Position lastCompactedPosition = compactedTopic.getCompactionHorizon().get(); - Entry lastCompactedEntry = compactedTopic.readLastEntryOfCompactedLedger().get(); - - Assert.assertTrue(PositionImpl.get(lastCompactedPosition.getLedgerId(), lastCompactedPosition.getEntryId()) - .compareTo(lastCompactedEntry.getLedgerId(), lastCompactedEntry.getEntryId()) >= 0); - - future.join(); - } - public ByteBuf extractPayload(RawMessage m) throws Exception { ByteBuf payloadAndMetadata = m.getHeadersAndPayload(); Commands.skipChecksumIfPresent(payloadAndMetadata); From 33ab210356543dd641d263995b8614674a1636a3 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Tue, 22 Aug 2023 18:07:09 +0800 Subject: [PATCH 249/494] [fix][broker] Fix can't stop phase-two of compaction even though messageId read reaches lastReadId (#20988) (cherry picked from commit 9e2195ce553ddbd6859b271a5ae0f2a23db02c44) --- .../pulsar/compaction/TwoPhaseCompactor.java | 1 + .../pulsar/compaction/CompactorTest.java | 95 ++++++++++++++----- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java index 21ccb281f0515..d71227d94f59f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java @@ -283,6 +283,7 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map promise.complete(null); } }); + return; } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java index e86be6a4db816..5f83ef97d32ca 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java @@ -19,13 +19,10 @@ package org.apache.pulsar.compaction; import static org.apache.pulsar.client.impl.RawReaderTest.extractKey; - import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import io.netty.buffer.ByteBuf; - import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; @@ -33,23 +30,30 @@ import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; - +import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawMessageImpl; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.Commands; +import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -65,7 +69,6 @@ public class CompactorTest extends MockedPulsarServiceBaseTest { protected Compactor compactor; - @BeforeMethod @Override public void setup() throws Exception { @@ -101,16 +104,17 @@ protected Compactor getCompactor() { return compactor; } - private List compactAndVerify(String topic, Map expected, boolean checkMetrics) throws Exception { + private List compactAndVerify(String topic, Map expected, boolean checkMetrics) + throws Exception { long compactedLedgerId = compact(topic); LedgerHandle ledger = bk.openLedger(compactedLedgerId, - Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, - Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); Assert.assertEquals(ledger.getLastAddConfirmed() + 1, // 0..lac - expected.size(), - "Should have as many entries as there is keys"); + expected.size(), + "Should have as many entries as there is keys"); List keys = new ArrayList<>(); Enumeration entries = ledger.readEntries(0, ledger.getLastAddConfirmed()); @@ -124,7 +128,7 @@ private List compactAndVerify(String topic, Map expected byte[] bytes = new byte[payload.readableBytes()]; payload.readBytes(bytes); Assert.assertEquals(bytes, expected.remove(key), - "Compacted version should match expected version"); + "Compacted version should match expected version"); m.close(); } if (checkMetrics) { @@ -148,17 +152,18 @@ public void testCompaction() throws Exception { final int numMessages = 1000; final int maxKeys = 10; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); Random r = new Random(0); for (int j = 0; j < numMessages; j++) { int keyIndex = r.nextInt(maxKeys); - String key = "key"+keyIndex; + String key = "key" + keyIndex; byte[] data = ("my-message-" + key + "-" + j).getBytes(); producer.newMessage() .key(key) @@ -173,10 +178,11 @@ public void testCompaction() throws Exception { public void testCompactAddCompact() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); @@ -210,10 +216,11 @@ public void testCompactAddCompact() throws Exception { public void testCompactedInOrder() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); producer.newMessage() .key("c") @@ -256,6 +263,50 @@ public void testPhaseOneLoopTimeConfiguration() { Assert.assertEquals(compactor.getPhaseOneLoopReadTimeoutInSeconds(), 60); } + @Test + public void testCompactedWithConcurrentSend() throws Exception { + String topic = "persistent://my-property/use/my-ns/testCompactedWithConcurrentSend"; + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + BookKeeper bk = pulsar.getBookKeeperClientFactory().create( + this.conf, null, null, Optional.empty(), null); + Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + + CompletableFuture future = CompletableFuture.runAsync(() -> { + for (int i = 0; i < 100; i++) { + try { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i).getBytes()).send(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + }); + + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + CompactedTopic compactedTopic = persistentTopic.getCompactedTopic(); + + Awaitility.await().untilAsserted(() -> { + long compactedLedgerId = compactor.compact(topic).join(); + Thread.sleep(300); + Optional compactedTopicContext = persistentTopic.getCompactedTopicContext(); + Assert.assertTrue(compactedTopicContext.isPresent()); + Assert.assertEquals(compactedTopicContext.get().ledger.getId(), compactedLedgerId); + }); + + Position lastCompactedPosition = compactedTopic.getCompactionHorizon().get(); + Entry lastCompactedEntry = compactedTopic.readLastEntryOfCompactedLedger().get(); + + Assert.assertTrue(PositionImpl.get(lastCompactedPosition.getLedgerId(), lastCompactedPosition.getEntryId()) + .compareTo(lastCompactedEntry.getLedgerId(), lastCompactedEntry.getEntryId()) >= 0); + + future.join(); + } + public ByteBuf extractPayload(RawMessage m) throws Exception { ByteBuf payloadAndMetadata = m.getHeadersAndPayload(); Commands.skipChecksumIfPresent(payloadAndMetadata); From 25f76efee83a297f04c3f9237bdbf73f268453c2 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 28 Aug 2023 09:42:22 +0800 Subject: [PATCH 250/494] [fix][broker] Make sure all inflight writes have finished before completion of compaction (#21067) (cherry picked from commit bb9c9b421b02c5f88fdd508ede387686065388f5) --- .../java/org/apache/pulsar/compaction/TwoPhaseCompactor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java index d71227d94f59f..886a523df0257 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java @@ -278,6 +278,8 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map } }); if (to.equals(id)) { + // make sure all inflight writes have finished + outstanding.acquire(MAX_OUTSTANDING); addFuture.whenComplete((res, exception2) -> { if (exception2 == null) { promise.complete(null); From f68589ea280eea63ee786236d2c2023b7001ecb6 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 23 Aug 2023 18:03:29 -0700 Subject: [PATCH 251/494] [fix] [broker] Producer is blocked on creation because backlog exceeded on topic, when dedup is enabled and no producer is there (#20951) (cherry picked from commit 30073dbac0e941869b43e090d2682935e8f094e5) --- .../broker/service/BacklogQuotaManager.java | 28 ++++++++++++- .../persistent/MessageDeduplication.java | 11 +++++ .../service/persistent/PersistentTopic.java | 4 ++ .../service/BacklogQuotaManagerTest.java | 40 ++++++++++++++++--- 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java index bc2541c802e63..6ad1697adfc39 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BacklogQuotaManager.java @@ -103,7 +103,10 @@ public void handleExceededBacklogQuota(PersistentTopic persistentTopic, BacklogQ break; case producer_exception: case producer_request_hold: - disconnectProducers(persistentTopic); + if (!advanceSlowestSystemCursor(persistentTopic)) { + // The slowest is not a system cursor. Disconnecting producers to put backpressure. + disconnectProducers(persistentTopic); + } break; default: break; @@ -268,4 +271,27 @@ private void disconnectProducers(PersistentTopic persistentTopic) { }); } + + /** + * Advances the slowest cursor if that is a system cursor. + * + * @param persistentTopic + * @return true if the slowest cursor is a system cursor + */ + private boolean advanceSlowestSystemCursor(PersistentTopic persistentTopic) { + + ManagedLedgerImpl mLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedCursor slowestConsumer = mLedger.getSlowestConsumer(); + if (slowestConsumer == null) { + return false; + } + + if (PersistentTopic.isDedupCursorName(slowestConsumer.getName())) { + persistentTopic.getMessageDeduplication().takeSnapshot(); + return true; + } + + // We may need to check other system cursors here : replicator, compaction + return false; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index 490be4a8876fc..d1b4b74945f8e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -27,6 +27,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback; @@ -130,6 +131,9 @@ public MessageDupUnknownException() { private final String replicatorPrefix; + + private final AtomicBoolean snapshotTaking = new AtomicBoolean(false); + public MessageDeduplication(PulsarService pulsar, PersistentTopic topic, ManagedLedger managedLedger) { this.pulsar = pulsar; this.topic = topic; @@ -406,6 +410,11 @@ private void takeSnapshot(Position position) { if (log.isDebugEnabled()) { log.debug("[{}] Taking snapshot of sequence ids map", topic.getName()); } + + if (!snapshotTaking.compareAndSet(false, true)) { + return; + } + Map snapshot = new TreeMap<>(); highestSequencedPersisted.forEach((producerName, sequenceId) -> { if (snapshot.size() < maxNumberOfProducers) { @@ -420,11 +429,13 @@ public void markDeleteComplete(Object ctx) { log.debug("[{}] Stored new deduplication snapshot at {}", topic.getName(), position); } lastSnapshotTimestamp = System.currentTimeMillis(); + snapshotTaking.set(false); } @Override public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { log.warn("[{}] Failed to store new deduplication snapshot at {}", topic.getName(), position); + snapshotTaking.set(false); } }, null); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 593f741ce39d7..ffb6828c8c311 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -194,6 +194,10 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal private final TopicName shadowSourceTopic; static final String DEDUPLICATION_CURSOR_NAME = "pulsar.dedup"; + + public static boolean isDedupCursorName(String name) { + return DEDUPLICATION_CURSOR_NAME.equals(name); + } private static final String TOPIC_EPOCH_PROPERTY_NAME = "pulsar.topic.epoch"; private static final double MESSAGE_EXPIRY_THRESHOLD = 1.5; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index f3463ee121d75..0ac5fdaef1599 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -127,8 +127,8 @@ void setup() throws Exception { config.setManagedLedgerMaxEntriesPerLedger(MAX_ENTRIES_PER_LEDGER); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); - config.setSystemTopicEnabled(false); - config.setTopicLevelPoliciesEnabled(false); + config.setSystemTopicEnabled(true); + config.setTopicLevelPoliciesEnabled(true); config.setForceDeleteNamespaceAllowed(true); pulsar = new PulsarService(config); @@ -1169,8 +1169,13 @@ public void testProducerException() throws Exception { assertTrue(gotException, "backlog exceeded exception did not occur"); } - @Test - public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { + @DataProvider(name = "dedupTestSet") + public static Object[][] dedupTestSet() { + return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; + } + + @Test(dataProvider = "dedupTestSet") + public void testProducerExceptionAndThenUnblockSizeQuota(boolean dedupTestSet) throws Exception { assertEquals(admin.namespaces().getBacklogQuotaMap("prop/quotahold"), new HashMap<>()); admin.namespaces().setBacklogQuota("prop/quotahold", @@ -1186,9 +1191,12 @@ public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { boolean gotException = false; Consumer consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe(); - byte[] content = new byte[1024]; Producer producer = createProducer(client, topic1); + + admin.topicPolicies().setDeduplicationStatus(topic1, dedupTestSet); + Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); + for (int i = 0; i < 10; i++) { producer.send(content); } @@ -1207,6 +1215,7 @@ public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { } assertTrue(gotException, "backlog exceeded exception did not occur"); + assertFalse(producer.isConnected()); // now remove backlog and ensure that producer is unblocked; TopicStats stats = getTopicStats(topic1); @@ -1223,14 +1232,33 @@ public void testProducerExceptionAndThenUnblockSizeQuota() throws Exception { Exception sendException = null; gotException = false; try { - for (int i = 0; i < 5; i++) { + for (int i = 0; i < 10; i++) { producer.send(content); + Message msg = consumer.receive(); + consumer.acknowledge(msg); } } catch (Exception e) { gotException = true; sendException = e; } + Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000); assertFalse(gotException, "unable to publish due to " + sendException); + + gotException = false; + long lastDisconnectedTimestamp = producer.getLastDisconnectedTimestamp(); + try { + // try to send over backlog quota and make sure it passes + producer.send(content); + producer.send(content); + } catch (PulsarClientException ce) { + assertTrue(ce instanceof PulsarClientException.ProducerBlockedQuotaExceededException + || ce instanceof PulsarClientException.TimeoutException, ce.getMessage()); + gotException = true; + sendException = ce; + } + assertFalse(gotException, "unable to publish due to " + sendException); + assertEquals(lastDisconnectedTimestamp, producer.getLastDisconnectedTimestamp()); + } @Test From dd151a5a5323076d2faa733bd96182ff685b56ee Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Tue, 22 Aug 2023 22:08:09 +0800 Subject: [PATCH 252/494] [fix][broker] Fix potential case cause retention policy not working on topic level (#21041) (cherry picked from commit d3a6df3c2a8f99ab3e5b7f6d826533ffd23328dc) --- .../service/persistent/PersistentTopic.java | 22 ++++++------- .../persistent/PersistentTopicTest.java | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index ffb6828c8c311..3e3363b8c5b75 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1535,7 +1535,6 @@ private void disposeTopic(CompletableFuture closeFuture) { CompletableFuture checkReplicationAndRetryOnFailure() { CompletableFuture result = new CompletableFuture(); checkReplication().thenAccept(res -> { - log.info("[{}] Policies updated successfully", topic); result.complete(null); }).exceptionally(th -> { log.error("[{}] Policies update failed {}, scheduled retry in {} seconds", topic, th.getMessage(), @@ -1555,7 +1554,8 @@ public CompletableFuture checkDeduplicationStatus() { return messageDeduplication.checkStatus(); } - private CompletableFuture checkPersistencePolicies() { + @VisibleForTesting + CompletableFuture checkPersistencePolicies() { TopicName topicName = TopicName.get(topic); CompletableFuture future = new CompletableFuture<>(); brokerService.getManagedLedgerConfig(topicName).thenAccept(config -> { @@ -3486,16 +3486,14 @@ public void onUpdate(TopicPolicies policies) { replicators.forEach((name, replicator) -> replicator.updateRateLimiter()); shadowReplicators.forEach((name, replicator) -> replicator.updateRateLimiter()); checkMessageExpiry(); - checkReplicationAndRetryOnFailure(); - - checkDeduplicationStatus(); - - preCreateSubscriptionForCompactionIfNeeded(); - - // update managed ledger config - checkPersistencePolicies(); - }).exceptionally(e -> { - Throwable t = e instanceof CompletionException ? e.getCause() : e; + }) + .thenCompose(__ -> checkReplicationAndRetryOnFailure()) + .thenCompose(__ -> checkDeduplicationStatus()) + .thenCompose(__ -> preCreateSubscriptionForCompactionIfNeeded()) + .thenCompose(__ -> checkPersistencePolicies()) + .thenAccept(__ -> log.info("[{}] Policies updated successfully", topic)) + .exceptionally(e -> { + Throwable t = FutureUtil.unwrapCompletionException(e); log.error("[{}] update topic policy error: {}", topic, t.getMessage(), t); return null; }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 412b8207e34c7..41704af0b8bb2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -60,6 +60,7 @@ import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.TopicPoliciesService; import org.apache.pulsar.broker.stats.PrometheusMetricsTest; import org.apache.pulsar.broker.stats.prometheus.PrometheusMetricsGenerator; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -77,7 +78,9 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicStats; import org.awaitility.Awaitility; import org.junit.Assert; @@ -603,4 +606,32 @@ public void testCreateTopicWithZombieReplicatorCursor(boolean topicLevelPolicy) return !topic.getManagedLedger().getCursors().iterator().hasNext(); }); } + + @Test + public void testCheckPersistencePolicies() throws Exception { + final String myNamespace = "prop/ns"; + admin.namespaces().createNamespace(myNamespace, Sets.newHashSet("test")); + final String topic = "persistent://" + myNamespace + "/testConfig" + UUID.randomUUID(); + conf.setForceDeleteNamespaceAllowed(true); + pulsarClient.newProducer().topic(topic).create().close(); + RetentionPolicies retentionPolicies = new RetentionPolicies(1, 1); + PersistentTopic persistentTopic = spy((PersistentTopic) pulsar.getBrokerService().getTopicIfExists(topic).get().get()); + TopicPoliciesService policiesService = spy(pulsar.getTopicPoliciesService()); + doReturn(policiesService).when(pulsar).getTopicPoliciesService(); + TopicPolicies policies = new TopicPolicies(); + policies.setRetentionPolicies(retentionPolicies); + doReturn(policies).when(policiesService).getTopicPoliciesIfExists(TopicName.get(topic)); + persistentTopic.onUpdate(policies); + verify(persistentTopic, times(1)).checkPersistencePolicies(); + Awaitility.await().untilAsserted(() -> { + assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionSizeInMB(), 1L); + assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionTimeMillis(), TimeUnit.MINUTES.toMillis(1)); + }); + // throw exception + doReturn(CompletableFuture.failedFuture(new RuntimeException())).when(persistentTopic).checkPersistencePolicies(); + policies.setRetentionPolicies(new RetentionPolicies(2, 2)); + persistentTopic.onUpdate(policies); + assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionSizeInMB(), 1L); + assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionTimeMillis(), TimeUnit.MINUTES.toMillis(1)); + } } From 2e534d236bb6b968eea915c8de7d6e496d442a63 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 28 Aug 2023 14:45:01 +0800 Subject: [PATCH 253/494] [improve] [broker] Improve cache handling for partitioned topic metadata when doing lookup (#21063) Motivation: If we set `allowAutoTopicCreationType` to `PARTITIONED`, the flow of the create topic progress is like the below: 1. `Client-side`: Lookup topic to get partitioned topic metadata to create a producer. 1. `Broker-side`: Create partitioned topic metadata. 1. `Broker-side`: response `{"partitions":3}`. 1. `Client-side`: Create separate connections for each partition of the topic. 1. `Broker-side`: Receive 3 connect requests and create 3 partition-topics. In the `step 2` above, the flow of the progress is like the below: 1. Check the policy of topic auto-creation( the policy is `{allowAutoTopicCreationType=PARTITIONED, defaultNumPartitions=3}` ) 1. Check the partitioned topic metadata already exists. 1. Try to create the partitioned topic metadata if it does not exist. 1. If created failed by the partitioned topic metadata already exists( maybe another broker is also creating now), read partitioned topic metadata from the metadata store and respond to the client. There is a race condition that makes the client get non-partitioned metadata of the topic: | time | `broker-1` | `broker-2` | | --- | --- | --- | | 1 | get policy: `PARTITIONED, 3` | get policy: `PARTITIONED, 3` | | 2 | check the partitioned topic metadata already exists | Check the partitioned topic metadata already exists | | 3 | Partitioned topic metadata does not exist, the metadata cache will cache an empty optional for the path | Partitioned topic metadata does not exist, the metadata cache will cache an empty optional for the path | | 4 | | succeed create the partitioned topic metadata | | 5 | Receive a ZK node changed event to invalidate the cache of the partitioned topic metadata | | 6 | Creating the metadata failed due to it already exists | | 7 | Read the partitioned topic metadata again | If `step-5` is executed later than `step-7`, `broker-1` will get an empty optional from the cache of the partitioned topic metadata and respond non-partitioned metadata to the client. **What thing would make the `step-5` is executed later than `step-7`?** Provide a scenario: Such as the issue that the PR https://github.com/apache/pulsar/pull/20303 fixed, it makes `zk operation` and `zk node changed notifications` executed in different threads: `main-thread of ZK client` and `metadata store thread`. Therefore, the mechanism of the lookup partitioned topic metadata is fragile and we need to optimize it. Modifications: Before reading the partitioned topic metadata again, refresh the cache first. (cherry picked from commit d099ac4fa2f217b9c5f0a5e660c83048e829c5d7) --- .../pulsar/broker/service/BrokerService.java | 15 ++- .../service/BrokerServiceChaosTest.java | 103 ++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 1f9996e77d917..2da3e73d8b68f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -3056,8 +3056,11 @@ public CompletableFuture fetchPartitionedTopicMetadata if (ex.getCause() instanceof MetadataStoreException .AlreadyExistsException) { + log.info("[{}] The partitioned topic is already" + + " created, try to refresh the cache and read" + + " again.", topicName); // The partitioned topic might be created concurrently - fetchPartitionedTopicMetadataAsync(topicName) + fetchPartitionedTopicMetadataAsync(topicName, true) .whenComplete((metadata2, ex2) -> { if (ex2 == null) { future.complete(metadata2); @@ -3066,6 +3069,9 @@ public CompletableFuture fetchPartitionedTopicMetadata } }); } else { + log.error("[{}] operation of creating partitioned" + + " topic metadata failed", + topicName, ex); future.completeExceptionally(ex); } return null; @@ -3111,9 +3117,14 @@ private CompletableFuture createDefaultPartitionedTopi } public CompletableFuture fetchPartitionedTopicMetadataAsync(TopicName topicName) { + return fetchPartitionedTopicMetadataAsync(topicName, false); + } + + public CompletableFuture fetchPartitionedTopicMetadataAsync(TopicName topicName, + boolean refreshCacheAndGet) { // gets the number of partitions from the configuration cache return pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() - .getPartitionedTopicMetadataAsync(topicName).thenApply(metadata -> { + .getPartitionedTopicMetadataAsync(topicName, refreshCacheAndGet).thenApply(metadata -> { // if the partitioned topic is not found in metadata, then the topic is not partitioned return metadata.orElseGet(() -> new PartitionedTopicMetadata()); }); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java new file mode 100644 index 0000000000000..4187364e46f65 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class BrokerServiceChaosTest extends CanReconnectZKClientPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testFetchPartitionedTopicMetadataWithCacheRefresh() throws Exception { + final String configMetadataStoreConnectString = + WhiteboxImpl.getInternalState(pulsar.getConfigurationMetadataStore(), "zkConnectString"); + final ZooKeeper anotherZKCli = new ZooKeeper(configMetadataStoreConnectString, 5000, null); + // Set policy of auto create topic to PARTITIONED. + final String ns = defaultTenant + "/ns_" + UUID.randomUUID().toString().replaceAll("-", ""); + final TopicName topicName1 = TopicName.get("persistent://" + ns + "/tp1"); + final TopicName topicName2 = TopicName.get("persistent://" + ns + "/tp2"); + admin.namespaces().createNamespace(ns); + AutoTopicCreationOverride autoTopicCreationOverride = + new AutoTopicCreationOverrideImpl.AutoTopicCreationOverrideImplBuilder().allowAutoTopicCreation(true) + .topicType(TopicType.PARTITIONED.toString()) + .defaultNumPartitions(3).build(); + admin.namespaces().setAutoTopicCreationAsync(ns, autoTopicCreationOverride); + // Make the cache of namespace policy is valid. + admin.namespaces().getAutoSubscriptionCreation(ns); + // Trigger the zk node "/admin/partitioned-topics/{namespace}/persistent" created. + admin.topics().createPartitionedTopic(topicName1.toString(), 2); + admin.topics().deletePartitionedTopic(topicName1.toString()); + + // Since there is no partitioned metadata created, the partitions count of metadata will be 0. + PartitionedTopicMetadata partitionedTopicMetadata1 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2).get(); + assertEquals(partitionedTopicMetadata1.partitions, 0); + + // Create the partitioned metadata by another zk client. + // Make a error to make the cache could not update. + makeLocalMetadataStoreKeepReconnect(); + anotherZKCli.create("/admin/partitioned-topics/" + ns + "/persistent/" + topicName2.getLocalName(), + "{\"partitions\":3}".getBytes(StandardCharsets.UTF_8), + ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + stopLocalMetadataStoreAlwaysReconnect(); + + // Get the partitioned metadata from cache, there is 90% chance that partitions count of metadata is 0. + PartitionedTopicMetadata partitionedTopicMetadata2 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2).get(); + // Note: If you want to reproduce the issue, you can perform validation on the next line. + // assertEquals(partitionedTopicMetadata2.partitions, 0); + + // Verify the new method will return a correct result. + PartitionedTopicMetadata partitionedTopicMetadata3 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2, true).get(); + assertEquals(partitionedTopicMetadata3.partitions, 3); + + // cleanup. + admin.topics().deletePartitionedTopic(topicName2.toString()); + anotherZKCli.close(); + } +} From c6c8496976211cb4ab512f4abb23c53a6daea4f9 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Mon, 28 Aug 2023 10:27:22 +0800 Subject: [PATCH 254/494] [fix][misc] Bump broker okio version to 3.4.0 (#21064) (cherry picked from commit 671cfb44ea8312fc3544d08793efba93ff8b7710) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 +++- pom.xml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index a1e77b91e2a74..11a7227a1520c 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -409,7 +409,9 @@ The Apache Software License, Version 2.0 * OkHttp3 - com.squareup.okhttp3-logging-interceptor-4.9.3.jar - com.squareup.okhttp3-okhttp-4.9.3.jar - * Okio - com.squareup.okio-okio-2.8.0.jar + * Okio + - com.squareup.okio-okio-3.4.0.jar + - com.squareup.okio-okio-jvm-3.4.0.jar * Javassist -- org.javassist-javassist-3.25.0-GA.jar * Kotlin Standard Lib - org.jetbrains.kotlin-kotlin-stdlib-1.6.0.jar diff --git a/pom.xml b/pom.xml index 83a6d1d129608..f1fead9205378 100644 --- a/pom.xml +++ b/pom.xml @@ -234,7 +234,7 @@ flexible messaging model and an intuitive client API. 18.0.0 4.9.3 - 2.8.0 + 3.4.0 1.6.0 1.0 From d4224635695afbb651a656ec698593ca98aaf677 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 29 Aug 2023 19:15:03 +0800 Subject: [PATCH 255/494] [fix][misc] Bump GRPC version to 1.55.3 to fix CVE (#21057) (cherry picked from commit 6ff83b6f8ab34bcb9045a2c249c8b14608dd965d) --- .../server/src/assemble/LICENSE.bin.txt | 36 +++++++++---------- pom.xml | 4 +-- pulsar-sql/presto-distribution/LICENSE | 20 +++++------ 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 11a7227a1520c..3d84a59a735ce 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -259,7 +259,7 @@ The Apache Software License, Version 2.0 - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar - * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar + * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.9.0.jar * Bitbucket -- org.bitbucket.b_c-jose4j-0.9.3.jar * Gson - com.google.code.gson-gson-2.8.9.jar @@ -420,24 +420,24 @@ The Apache Software License, Version 2.0 - org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.6.0.jar - org.jetbrains-annotations-13.0.jar * gRPC - - io.grpc-grpc-all-1.45.1.jar - - io.grpc-grpc-auth-1.45.1.jar - - io.grpc-grpc-context-1.45.1.jar - - io.grpc-grpc-core-1.45.1.jar - - io.grpc-grpc-netty-1.45.1.jar - - io.grpc-grpc-protobuf-1.45.1.jar - - io.grpc-grpc-protobuf-lite-1.45.1.jar - - io.grpc-grpc-stub-1.45.1.jar - - io.grpc-grpc-alts-1.45.1.jar - - io.grpc-grpc-api-1.45.1.jar - - io.grpc-grpc-grpclb-1.45.1.jar - - io.grpc-grpc-netty-shaded-1.45.1.jar - - io.grpc-grpc-services-1.45.1.jar - - io.grpc-grpc-xds-1.45.1.jar - - io.grpc-grpc-rls-1.45.1.jar + - io.grpc-grpc-all-1.55.3.jar + - io.grpc-grpc-auth-1.55.3.jar + - io.grpc-grpc-context-1.55.3.jar + - io.grpc-grpc-core-1.55.3.jar + - io.grpc-grpc-netty-1.55.3.jar + - io.grpc-grpc-protobuf-1.55.3.jar + - io.grpc-grpc-protobuf-lite-1.55.3.jar + - io.grpc-grpc-stub-1.55.3.jar + - io.grpc-grpc-alts-1.55.3.jar + - io.grpc-grpc-api-1.55.3.jar + - io.grpc-grpc-grpclb-1.55.3.jar + - io.grpc-grpc-netty-shaded-1.55.3.jar + - io.grpc-grpc-services-1.55.3.jar + - io.grpc-grpc-xds-1.55.3.jar + - io.grpc-grpc-rls-1.55.3.jar - com.google.auto.service-auto-service-annotations-1.0.jar * Perfmark - - io.perfmark-perfmark-api-0.19.0.jar + - io.perfmark-perfmark-api-0.26.0.jar * OpenCensus - io.opencensus-opencensus-api-0.28.0.jar - io.opencensus-opencensus-contrib-http-util-0.28.0.jar @@ -487,7 +487,7 @@ The Apache Software License, Version 2.0 - com.google.http-client-google-http-client-gson-1.41.0.jar - com.google.http-client-google-http-client-1.41.0.jar - com.google.auto.value-auto-value-annotations-1.9.jar - - com.google.re2j-re2j-1.5.jar + - com.google.re2j-re2j-1.6.jar * Jetcd - io.etcd-jetcd-common-0.5.11.jar - io.etcd-jetcd-core-0.5.11.jar diff --git a/pom.xml b/pom.xml index f1fead9205378..d5d2fdcd00f69 100644 --- a/pom.xml +++ b/pom.xml @@ -164,9 +164,9 @@ flexible messaging model and an intuitive client API. 0.5.0 3.19.6 ${protobuf3.version} - 1.45.1 + 1.55.3 1.41.0 - 0.19.0 + 0.26.0 ${grpc.version} 2.8.9 1.2.1 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index e70bcee56ecae..cfbd2e7d59253 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -263,14 +263,14 @@ The Apache Software License, Version 2.0 - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar * GRPC - - grpc-api-1.45.1.jar - - grpc-context-1.45.1.jar - - grpc-core-1.45.1.jar - - grpc-grpclb-1.45.1.jar - - grpc-netty-1.45.1.jar - - grpc-protobuf-1.45.1.jar - - grpc-protobuf-lite-1.45.1.jar - - grpc-stub-1.45.1.jar + - grpc-api-1.55.3.jar + - grpc-context-1.55.3.jar + - grpc-core-1.55.3.jar + - grpc-grpclb-1.55.3.jar + - grpc-netty-1.55.3.jar + - grpc-protobuf-1.55.3.jar + - grpc-protobuf-lite-1.55.3.jar + - grpc-stub-1.55.3.jar * JEtcd - jetcd-common-0.5.11.jar - jetcd-core-0.5.11.jar @@ -475,7 +475,7 @@ The Apache Software License, Version 2.0 * Swagger - swagger-annotations-1.6.10.jar * Perfmark - - perfmark-api-0.19.0.jar + - perfmark-api-0.26.0.jar * Annotations - auto-service-annotations-1.0.jar * RabbitMQ Java Client @@ -490,7 +490,7 @@ Protocol Buffers License * Protocol Buffers - protobuf-java-3.19.6.jar - protobuf-java-util-3.19.6.jar - - proto-google-common-protos-2.0.1.jar + - proto-google-common-protos-2.9.0.jar BSD 3-clause "New" or "Revised" License * RE2J TD -- re2j-td-1.4.jar From 3b085eab913c6afc69d5c624e18889fe101127e1 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Wed, 30 Aug 2023 00:23:52 +0800 Subject: [PATCH 256/494] [fix][client] Fix consumer can't consume resent chunked messages (#21070) ### Motivation Current, when the producer resend the chunked message like this: - M1: UUID: 0, ChunkID: 0 - M2: UUID: 0, ChunkID: 0 // Resend the first chunk - M3: UUID: 0, ChunkID: 1 When the consumer received the M2, it will find that it's already tracking the UUID:0 chunked messages, and will then discard the message M1 and M2. This will lead to unable to consume the whole chunked message even though it's already persisted in the Pulsar topic. Here is the code logic: https://github.com/apache/pulsar/blob/44a055b8a55078bcf93f4904991598541aa6c1ee/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java#L1436-L1482 The bug can be easily reproduced using the testcase `testResendChunkMessages` introduced by this PR. ### Modifications - When receiving the new duplicated first chunk of a chunked message, the consumer discard the current chunked message context and create a new context to track the following messages. For the case mentioned in Motivation, the M1 will be released and the consumer will assemble M2 and M3 as the chunked message. (cherry picked from commit eb2e3a258b971cfeeb22f1cec254cafb49d0ae40) --- .../client/impl/MessageChunkingTest.java | 41 +++++++++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 12 +++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java index 2797d66f7fe21..dffa003524864 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java @@ -356,6 +356,47 @@ public void testMaxPendingChunkMessages() throws Exception { assertNull(consumer.receive(5, TimeUnit.SECONDS)); } + @Test + public void testResendChunkMessages() throws Exception { + log.info("-- Starting {} test --", methodName); + final String topicName = "persistent://my-property/my-ns/testResendChunkMessages"; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName("my-subscriber-name") + .maxPendingChunkedMessage(10) + .autoAckOldestChunkedMessageOnQueueFull(true) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .chunkMaxMessageSize(100) + .enableChunking(true) + .enableBatching(false) + .create(); + + sendSingleChunk(producer, "0", 0, 2); + + sendSingleChunk(producer, "0", 0, 2); // Resending the first chunk + sendSingleChunk(producer, "1", 0, 3); // This is for testing the interwoven chunked message + sendSingleChunk(producer, "1", 1, 3); + sendSingleChunk(producer, "1", 0, 3); // Resending the UUID-1 chunked message + + sendSingleChunk(producer, "0", 1, 2); + + Message receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-0-0|chunk-0-1|"); + consumer.acknowledge(receivedMsg); + + sendSingleChunk(producer, "1", 1, 3); + sendSingleChunk(producer, "1", 2, 3); + + receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-1-0|chunk-1-1|chunk-1-2|"); + consumer.acknowledge(receivedMsg); + } + /** * Validate that chunking is not supported with batching and non-persistent topic * diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index bb27562188d2f..ac910c4d1c9b8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1425,7 +1425,17 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m ChunkedMessageCtx chunkedMsgCtx = chunkedMessagesMap.get(msgMetadata.getUuid()); - if (msgMetadata.getChunkId() == 0 && chunkedMsgCtx == null) { + if (msgMetadata.getChunkId() == 0) { + if (chunkedMsgCtx != null) { + // The first chunk of a new chunked-message received before receiving other chunks of previous + // chunked-message + // so, remove previous chunked-message from map and release buffer + if (chunkedMsgCtx.chunkedMsgBuffer != null) { + ReferenceCountUtil.safeRelease(chunkedMsgCtx.chunkedMsgBuffer); + } + chunkedMsgCtx.recycle(); + chunkedMessagesMap.remove(msgMetadata.getUuid()); + } pendingChunkedMessageCount++; if (maxPendingChunkedMessage > 0 && pendingChunkedMessageCount > maxPendingChunkedMessage) { removeOldestPendingChunkedMessage(); From fab85bdaf406441f40c8bd5349296566599be445 Mon Sep 17 00:00:00 2001 From: Zike Yang Date: Thu, 31 Aug 2023 07:44:34 +0800 Subject: [PATCH 257/494] [fix][client] Fix cannot retry chunk messages and send to DLQ (#21048) (cherry picked from commit 99e3fea2e3adecb3346d1fc8e5422a3fde4236e3) --- .../client/api/DeadLetterTopicTest.java | 48 +++++++++++++++--- .../pulsar/client/impl/ConsumerImpl.java | 6 ++- .../pulsar/client/impl/MessageIdAdvUtils.java | 3 ++ .../pulsar/client/impl/ProducerImpl.java | 6 +++ .../client/impl/UnAckedMessageTracker.java | 7 ++- .../impl/UnAckedMessageTrackerTest.java | 49 ++++++++++++++++++- 6 files changed, 107 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java index 5e3731fbf24fc..2a0cb3187d208 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DeadLetterTopicTest.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -46,9 +47,10 @@ import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -@Test(groups = "flaky") +@Test(groups = "broker-impl") public class DeadLetterTopicTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(DeadLetterTopicTest.class); @@ -56,6 +58,7 @@ public class DeadLetterTopicTest extends ProducerConsumerBase { @BeforeMethod(alwaysRun = true) @Override protected void setup() throws Exception { + this.conf.setMaxMessageSize(5 * 1024); super.internalSetup(); super.producerBaseSetup(); } @@ -66,6 +69,15 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + private String createMessagePayload(int size) { + StringBuilder str = new StringBuilder(); + Random rand = new Random(); + for (int i = 0; i < size; i++) { + str.append(rand.nextInt(10)); + } + return str.toString(); + } + @Test public void testDeadLetterTopicWithMessageKey() throws Exception { final String topic = "persistent://my-property/my-ns/dead-letter-topic"; @@ -125,9 +137,13 @@ public void testDeadLetterTopicWithMessageKey() throws Exception { consumer.close(); } + @DataProvider(name = "produceLargeMessages") + public Object[][] produceLargeMessages() { + return new Object[][] { { false }, { true } }; + } - @Test(groups = "quarantine") - public void testDeadLetterTopic() throws Exception { + @Test(dataProvider = "produceLargeMessages") + public void testDeadLetterTopic(boolean produceLargeMessages) throws Exception { final String topic = "persistent://my-property/my-ns/dead-letter-topic"; final int maxRedeliveryCount = 2; @@ -154,28 +170,44 @@ public void testDeadLetterTopic() throws Exception { Producer producer = pulsarClient.newProducer(Schema.BYTES) .topic(topic) + .enableChunking(produceLargeMessages) + .enableBatching(!produceLargeMessages) .create(); + Map messageContent = new HashMap<>(); + for (int i = 0; i < sendMessages; i++) { - producer.send(String.format("Hello Pulsar [%d]", i).getBytes()); + String data; + if (!produceLargeMessages) { + data = String.format("Hello Pulsar [%d]", i); + } else { + data = createMessagePayload(1024 * 10); + } + producer.newMessage().key(String.valueOf(i)).value(data.getBytes()).send(); + messageContent.put(i, data); } producer.close(); int totalReceived = 0; do { - Message message = consumer.receive(); - log.info("consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + Message message = consumer.receive(5, TimeUnit.SECONDS); + assertNotNull(message, "The consumer should be able to receive messages."); + log.info("consumer received message : {}", message.getMessageId()); totalReceived++; } while (totalReceived < sendMessages * (maxRedeliveryCount + 1)); int totalInDeadLetter = 0; do { - Message message = deadLetterConsumer.receive(); - log.info("dead letter consumer received message : {} {}", message.getMessageId(), new String(message.getData())); + Message message = deadLetterConsumer.receive(5, TimeUnit.SECONDS); + assertNotNull(message, "the deadLetterConsumer should receive messages."); + assertEquals(new String(message.getData()), messageContent.get(Integer.parseInt(message.getKey()))); + messageContent.remove(Integer.parseInt(message.getKey())); + log.info("dead letter consumer received message : {}", message.getMessageId()); deadLetterConsumer.acknowledge(message); totalInDeadLetter++; } while (totalInDeadLetter < sendMessages); + assertTrue(messageContent.isEmpty()); deadLetterConsumer.close(); consumer.close(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index ac910c4d1c9b8..47aecd8052c29 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -605,6 +605,7 @@ protected CompletableFuture doReconsumeLater(Message message, AckType a retryLetterProducer = client.newProducer(Schema.AUTO_PRODUCE_BYTES(schema)) .topic(this.deadLetterPolicy.getRetryLetterTopic()) .enableBatching(false) + .enableChunking(true) .blockIfQueueFull(false) .create(); } @@ -1452,7 +1453,8 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m if (chunkedMsgCtx == null || chunkedMsgCtx.chunkedMsgBuffer == null || msgMetadata.getChunkId() != (chunkedMsgCtx.lastChunkedMessageId + 1)) { // means we lost the first chunk: should never happen - log.info("Received unexpected chunk messageId {}, last-chunk-id{}, chunkId = {}", msgId, + log.info("[{}] [{}] Received unexpected chunk messageId {}, last-chunk-id = {}, chunkId = {}", topic, + subscription, msgId, (chunkedMsgCtx != null ? chunkedMsgCtx.lastChunkedMessageId : null), msgMetadata.getChunkId()); if (chunkedMsgCtx != null) { if (chunkedMsgCtx.chunkedMsgBuffer != null) { @@ -2096,6 +2098,8 @@ private void initDeadLetterProducerIfNeeded() { .initialSubscriptionName(this.deadLetterPolicy.getInitialSubscriptionName()) .topic(this.deadLetterPolicy.getDeadLetterTopic()) .blockIfQueueFull(false) + .enableBatching(false) + .enableChunking(true) .createAsync(); } } finally { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java index c8b18524ec052..a0d1446ba3d55 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdAdvUtils.java @@ -64,6 +64,9 @@ static boolean isBatch(MessageIdAdv msgId) { } static MessageIdAdv discardBatch(MessageId messageId) { + if (messageId instanceof ChunkMessageIdImpl) { + return (MessageIdAdv) messageId; + } MessageIdAdv msgId = (MessageIdAdv) messageId; return new MessageIdImpl(msgId.getLedgerId(), msgId.getEntryId(), msgId.getPartitionIndex()); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java index 267b06649d719..8a30190b3a274 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ProducerImpl.java @@ -695,6 +695,12 @@ private void serializeAndSendMessage(MessageImpl msg, op = OpSendMsg.create(msg, null, sequenceId, callback); final MessageMetadata finalMsgMetadata = msgMetadata; op.rePopulate = () -> { + if (msgMetadata.hasChunkId()) { + // The message metadata is shared between all chunks in a large message + // We need to reset the chunk id for each call of this method + // It's safe to do that because there is only 1 thread to manipulate this message metadata + finalMsgMetadata.setChunkId(chunkId); + } op.cmd = sendMessage(producerId, sequenceId, numMessages, messageId, finalMsgMetadata, encryptedPayload); }; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java index 534f33350267d..220771e426fec 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/UnAckedMessageTracker.java @@ -138,8 +138,11 @@ public void run(Timeout t) throws Exception { if (!headPartition.isEmpty()) { log.info("[{}] {} messages will be re-delivered", consumerBase, headPartition.size()); headPartition.forEach(messageId -> { - addChunkedMessageIdsAndRemoveFromSequenceMap(messageId, messageIds, consumerBase); - messageIds.add(messageId); + if (messageId instanceof ChunkMessageIdImpl) { + addChunkedMessageIdsAndRemoveFromSequenceMap(messageId, messageIds, consumerBase); + } else { + messageIds.add(messageId); + } messageIdPartitionMap.remove(messageId); }); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java index 4ccc514e8e7f1..f6c668703d9db 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java @@ -29,12 +29,15 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timer; import io.netty.util.concurrent.DefaultThreadFactory; - +import java.time.Duration; import java.util.HashSet; import java.util.concurrent.TimeUnit; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.awaitility.Awaitility; import org.testng.annotations.Test; public class UnAckedMessageTrackerTest { @@ -77,4 +80,48 @@ public void testAddAndRemove() { timer.stop(); } + @Test + public void testTrackChunkedMessageId() { + PulsarClientImpl client = mock(PulsarClientImpl.class); + Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), + 1, TimeUnit.MILLISECONDS); + when(client.timer()).thenReturn(timer); + + ConsumerBase consumer = mock(ConsumerBase.class); + doNothing().when(consumer).onAckTimeoutSend(any()); + doNothing().when(consumer).redeliverUnacknowledgedMessages(any()); + ConsumerConfigurationData conf = new ConsumerConfigurationData<>(); + conf.setAckTimeoutMillis(1000); + conf.setTickDurationMillis(1000); + UnAckedMessageTracker tracker = new UnAckedMessageTracker(client, consumer, conf); + + assertTrue(tracker.isEmpty()); + assertEquals(tracker.size(), 0); + + // Build chunked message ID + MessageIdImpl[] chunkMsgIds = new MessageIdImpl[5]; + for (int i = 0; i < 5; i++) { + chunkMsgIds[i] = new MessageIdImpl(1L, i, -1); + } + ChunkMessageIdImpl chunkedMessageId = + new ChunkMessageIdImpl(chunkMsgIds[0], chunkMsgIds[chunkMsgIds.length - 1]); + + consumer.unAckedChunkedMessageIdSequenceMap = + ConcurrentOpenHashMap.newBuilder().build(); + consumer.unAckedChunkedMessageIdSequenceMap.put(chunkedMessageId, chunkMsgIds); + + // Redeliver chunked message + tracker.add(chunkedMessageId); + + Awaitility.await() + .pollInterval(Duration.ofMillis(200)) + .atMost(Duration.ofSeconds(3)) + .untilAsserted(() -> assertEquals(tracker.size(), 0)); + + // Assert that all chunk message ID are removed from unAckedChunkedMessageIdSequenceMap + assertEquals(consumer.unAckedChunkedMessageIdSequenceMap.size(), 0); + + timer.stop(); + } + } From 63ea2d88e4da2f7cfa038475d3d80fe3ab89c1a4 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 1 Sep 2023 02:37:00 +0800 Subject: [PATCH 258/494] [improve] [ml] Persist mark deleted ops to ZK if create cursor ledger was failed (#20935) The progress Persist mark deleted position is like this: - persist to BK - If failed to persist to BK, try to persist to ZK But in the current implementation: if the cursor ledger was created failed, Pulsar will not try to persist to ZK. It makes if the cursor ledger created fails, a lot of ack records can not be persisted, and we will get a lot of repeat consumption after the BK recover. Modifications: Try to persist the mark deleted position to ZK if the cursor ledger was created failed (cherry picked from commit 843b8307f44cd5e3a2d59ab93cc6b766f0c4ce0f) --- .../mledger/impl/ManagedCursorImpl.java | 77 +++++++------ .../mledger/impl/ManagedLedgerImpl.java | 28 +++++ .../mledger/impl/ManagedCursorTest.java | 102 ++++++++++++++++++ .../mledger/impl/ManagedLedgerTest.java | 64 +++++++++++ 4 files changed, 241 insertions(+), 30 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index e5d784d40d48e..965e078bf02a8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -2100,7 +2100,7 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) { } }); - persistPositionToLedger(cursorLedger, mdEntry, new VoidCallback() { + VoidCallback cb = new VoidCallback() { @Override public void operationComplete() { if (log.isDebugEnabled()) { @@ -2152,7 +2152,18 @@ public void operationFailed(ManagedLedgerException exception) { mdEntry.triggerFailed(exception); } - }); + }; + + if (State.NoLedger.equals(STATE_UPDATER.get(this))) { + if (ledger.isNoMessagesAfterPos(mdEntry.newPosition)) { + persistPositionToMetaStore(mdEntry, cb); + } else { + mdEntry.callback.markDeleteFailed(new ManagedLedgerException("Create new cursor ledger failed"), + mdEntry.ctx); + } + } else { + persistPositionToLedger(cursorLedger, mdEntry, cb); + } } @Override @@ -2798,16 +2809,15 @@ public void operationComplete() { @Override public void operationFailed(ManagedLedgerException exception) { - log.error("[{}][{}] Metadata ledger creation failed", ledger.getName(), name, exception); + log.error("[{}][{}] Metadata ledger creation failed {}, try to persist the position in the metadata" + + " store.", ledger.getName(), name, exception); synchronized (pendingMarkDeleteOps) { - while (!pendingMarkDeleteOps.isEmpty()) { - MarkDeleteEntry entry = pendingMarkDeleteOps.poll(); - entry.callback.markDeleteFailed(exception, entry.ctx); - } - // At this point we don't have a ledger ready STATE_UPDATER.set(ManagedCursorImpl.this, State.NoLedger); + // Note: if the stat is NoLedger, will persist the mark deleted position to metadata store. + // Before giving up, try to persist the position in the metadata store. + flushPendingMarkDeletes(); } } }); @@ -3074,32 +3084,39 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin // in the meantime the mark-delete will be queued. STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger); - mbean.persistToLedger(false); - // Before giving up, try to persist the position in the metadata store - persistPositionMetaStore(-1, position, mdEntry.properties, new MetaStoreCallback() { - @Override - public void operationComplete(Void result, Stat stat) { - if (log.isDebugEnabled()) { - log.debug( - "[{}][{}] Updated cursor in meta store after previous failure in ledger at position" - + " {}", ledger.getName(), name, position); - } - mbean.persistToZookeeper(true); - callback.operationComplete(); - } - - @Override - public void operationFailed(MetaStoreException e) { - log.warn("[{}][{}] Failed to update cursor in meta store after previous failure in ledger: {}", - ledger.getName(), name, e.getMessage()); - mbean.persistToZookeeper(false); - callback.operationFailed(createManagedLedgerException(rc)); - } - }, true); + // Before giving up, try to persist the position in the metadata store. + persistPositionToMetaStore(mdEntry, callback); } }, null); } + void persistPositionToMetaStore(MarkDeleteEntry mdEntry, final VoidCallback callback) { + final PositionImpl newPosition = mdEntry.newPosition; + STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger); + mbean.persistToLedger(false); + // Before giving up, try to persist the position in the metadata store + persistPositionMetaStore(-1, newPosition, mdEntry.properties, new MetaStoreCallback() { + @Override + public void operationComplete(Void result, Stat stat) { + if (log.isDebugEnabled()) { + log.debug( + "[{}][{}] Updated cursor in meta store after previous failure in ledger at position" + + " {}", ledger.getName(), name, newPosition); + } + mbean.persistToZookeeper(true); + callback.operationComplete(); + } + + @Override + public void operationFailed(MetaStoreException e) { + log.warn("[{}][{}] Failed to update cursor in meta store after previous failure in ledger: {}", + ledger.getName(), name, e.getMessage()); + mbean.persistToZookeeper(false); + callback.operationFailed(createManagedLedgerException(e)); + } + }, true); + } + boolean shouldCloseLedger(LedgerHandle lh) { long now = clock.millis(); if (ledger.getFactory().isMetadataServiceAvailable() diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index c31a0c38cd395..89c18f4b834f3 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -3552,6 +3552,34 @@ public PositionImpl getPositionAfterN(final PositionImpl startPosition, long n, return positionToReturn; } + public boolean isNoMessagesAfterPos(PositionImpl pos) { + PositionImpl lac = (PositionImpl) getLastConfirmedEntry(); + return isNoMessagesAfterPosForSpecifiedLac(lac, pos); + } + + private boolean isNoMessagesAfterPosForSpecifiedLac(PositionImpl specifiedLac, PositionImpl pos) { + if (pos.compareTo(specifiedLac) >= 0) { + return true; + } + if (specifiedLac.getEntryId() < 0) { + // Calculate the meaningful LAC. + PositionImpl actLac = getPreviousPosition(specifiedLac); + if (actLac.getEntryId() >= 0) { + return pos.compareTo(actLac) >= 0; + } else { + // If the actual LAC is still not meaningful. + if (actLac.equals(specifiedLac)) { + // No entries in maneged ledger. + return true; + } else { + // Continue to find a valid LAC. + return isNoMessagesAfterPosForSpecifiedLac(actLac, pos); + } + } + } + return false; + } + /** * Get the entry position that come before the specified position in the message stream, using information from the * ledger list and each ledger entries count. diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 1b1b5534256f9..627ae73d928bd 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -229,6 +229,97 @@ void readTwice() throws Exception { entries.forEach(Entry::release); } + @Test + void testPersistentMarkDeleteIfCreateCursorLedgerFailed() throws Exception { + final int entryCount = 10; + final String cursorName = "c1"; + final String mlName = "ml_test"; + final ManagedLedgerConfig mlConfig = new ManagedLedgerConfig().setMaxEntriesPerLedger(1); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + + ManagedCursor cursor = ml.openCursor("c1"); + Position lastEntry = null; + for (int i = 0; i < 10; i++) { + lastEntry = ml.addEntry(("entry-" + i).getBytes(Encoding)); + } + + // Mock cursor ledger create failed. + bkc.failNow(BKException.Code.NoBookieAvailableException); + + cursor.markDelete(lastEntry); + + // Assert persist mark deleted position to ZK was successful. + PositionImpl slowestReadPosition = ml.getCursors().getSlowestReaderPosition(); + assertTrue(slowestReadPosition.getLedgerId() >= lastEntry.getLedgerId()); + assertTrue(slowestReadPosition.getEntryId() >= lastEntry.getEntryId()); + assertEquals(cursor.getStats().getPersistLedgerSucceed(), 0); + assertTrue(cursor.getStats().getPersistZookeeperSucceed() > 0); + assertEquals(cursor.getPersistentMarkDeletedPosition(), lastEntry); + + // Verify the mark delete position can be recovered properly. + ml.close(); + ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + ManagedCursorImpl cursorRecovered = (ManagedCursorImpl) ml.openCursor(cursorName); + assertEquals(cursorRecovered.getPersistentMarkDeletedPosition(), lastEntry); + + // cleanup. + ml.delete(); + } + + @Test + void testPersistentMarkDeleteIfSwitchCursorLedgerFailed() throws Exception { + final int entryCount = 10; + final String cursorName = "c1"; + final String mlName = "ml_test"; + final ManagedLedgerConfig mlConfig = new ManagedLedgerConfig().setMaxEntriesPerLedger(1); + ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + + final ManagedCursorImpl cursor = (ManagedCursorImpl) ml.openCursor(cursorName); + ArrayList positions = new ArrayList<>(); + for (int i = 0; i < entryCount; i++) { + positions.add(ml.addEntry(("entry-" + i).getBytes(Encoding))); + } + // Trigger the cursor ledger creating. + cursor.markDelete(positions.get(0)); + assertTrue(cursor.getStats().getPersistLedgerSucceed() > 0); + + // Mock cursor ledger write failed. + bkc.addEntryFailAfter(0, BKException.Code.NoBookieAvailableException); + // Trigger a failed writing of the cursor ledger, then wait the stat of cursor to be "NoLedger". + // This time ZK will be written due to a failure to write BK. + cursor.markDelete(positions.get(1)); + Awaitility.await().untilAsserted(() -> { + assertEquals(cursor.getState(), "NoLedger"); + }); + assertTrue(cursor.getStats().getPersistLedgerErrors() > 0); + long persistZookeeperSucceed1 = cursor.getStats().getPersistZookeeperSucceed(); + assertTrue(persistZookeeperSucceed1 > 0); + + // Mock cursor ledger create failed. + bkc.failNow(BKException.Code.NoBookieAvailableException); + // Verify the cursor status will be persistent to ZK even if the cursor ledger creation always fails. + // This time ZK will be written due to catch up. + Position lastEntry = positions.get(entryCount -1); + cursor.markDelete(lastEntry); + long persistZookeeperSucceed2 = cursor.getStats().getPersistZookeeperSucceed(); + assertTrue(persistZookeeperSucceed2 > persistZookeeperSucceed1); + + // Assert persist mark deleted position to ZK was successful. + PositionImpl slowestReadPosition = ml.getCursors().getSlowestReaderPosition(); + assertTrue(slowestReadPosition.getLedgerId() >= lastEntry.getLedgerId()); + assertTrue(slowestReadPosition.getEntryId() >= lastEntry.getEntryId()); + assertEquals(cursor.getPersistentMarkDeletedPosition(), lastEntry); + + // Verify the mark delete position can be recovered properly. + ml.close(); + ml = (ManagedLedgerImpl) factory.open(mlName, mlConfig); + ManagedCursorImpl cursorRecovered = (ManagedCursorImpl) ml.openCursor(cursorName); + assertEquals(cursorRecovered.getPersistentMarkDeletedPosition(), lastEntry); + + // cleanup. + ml.delete(); + } + @Test(timeOut = 20000) void readWithCacheDisabled() throws Exception { ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig(); @@ -1421,6 +1512,17 @@ void errorRecoveringCursor2() throws Exception { ledger = factory2.open("my_test_ledger"); ManagedCursor cursor = ledger.openCursor("c1"); Position position = ledger.addEntry("test".getBytes()); + // Make persist zk fail once. + AtomicInteger persistZKTimes = new AtomicInteger(); + metadataStore.failConditional(new MetadataStoreException.BadVersionException("mock ex"), (type, path) -> { + if (FaultInjectionMetadataStore.OperationType.PUT.equals(type) + && path.equals("/managed-ledgers/my_test_ledger/c1")) { + if (persistZKTimes.incrementAndGet() == 1) { + return true; + } + } + return false; + }); try { cursor.markDelete(position); fail("should have failed"); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 70ddbb9998fd8..5fc2da22b661e 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -3968,6 +3968,70 @@ public void testGetEnsemblesAsync() throws Exception { Assert.assertFalse(managedLedger.ledgerCache.containsKey(lastLedger)); } + @Test + public void testIsNoMessagesAfterPos() throws Exception { + final byte[] data = new byte[]{1,2,3}; + final String cursorName = "c1"; + final String mlName = UUID.randomUUID().toString().replaceAll("-", ""); + final ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open(mlName); + final ManagedCursor managedCursor = ml.openCursor(cursorName); + + // One ledger. + PositionImpl p1 = (PositionImpl) ml.addEntry(data); + PositionImpl p2 = (PositionImpl) ml.addEntry(data); + PositionImpl p3 = (PositionImpl) ml.addEntry(data); + assertFalse(ml.isNoMessagesAfterPos(p1)); + assertFalse(ml.isNoMessagesAfterPos(p2)); + assertTrue(ml.isNoMessagesAfterPos(p3)); + assertTrue(ml.isNoMessagesAfterPos(PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1))); + assertTrue(ml.isNoMessagesAfterPos(PositionImpl.get(p3.getLedgerId() + 1, -1))); + + // More than one ledger. + ml.ledgerClosed(ml.currentLedger); + PositionImpl p4 = (PositionImpl) ml.addEntry(data); + PositionImpl p5 = (PositionImpl) ml.addEntry(data); + PositionImpl p6 = (PositionImpl) ml.addEntry(data); + assertFalse(ml.isNoMessagesAfterPos(p1)); + assertFalse(ml.isNoMessagesAfterPos(p2)); + assertFalse(ml.isNoMessagesAfterPos(p3)); + assertFalse(ml.isNoMessagesAfterPos(p4)); + assertFalse(ml.isNoMessagesAfterPos(p5)); + assertTrue(ml.isNoMessagesAfterPos(p6)); + assertTrue(ml.isNoMessagesAfterPos(PositionImpl.get(p6.getLedgerId(), p6.getEntryId() + 1))); + assertTrue(ml.isNoMessagesAfterPos(PositionImpl.get(p6.getLedgerId() + 1, -1))); + + // Switch ledger and make the entry id of Last confirmed entry is -1; + ml.ledgerClosed(ml.currentLedger); + ml.createLedgerAfterClosed(); + Awaitility.await().untilAsserted(() -> { + assertEquals(ml.currentLedgerEntries, 0); + }); + ml.lastConfirmedEntry = PositionImpl.get(ml.currentLedger.getId(), -1); + assertFalse(ml.isNoMessagesAfterPos(p5)); + assertTrue(ml.isNoMessagesAfterPos(p6)); + assertTrue(ml.isNoMessagesAfterPos(PositionImpl.get(p6.getLedgerId(), p6.getEntryId() + 1))); + assertTrue(ml.isNoMessagesAfterPos(PositionImpl.get(p6.getLedgerId() + 1, -1))); + + // Trim ledgers to make there is no entries in ML. + ml.deleteCursor(cursorName); + CompletableFuture future = new CompletableFuture<>(); + ml.trimConsumedLedgersInBackground(true, future); + future.get(); + assertEquals(ml.ledgers.size(), 1); + assertEquals(ml.lastConfirmedEntry.getEntryId(), -1); + assertTrue(ml.isNoMessagesAfterPos(p1)); + assertTrue(ml.isNoMessagesAfterPos(p2)); + assertTrue(ml.isNoMessagesAfterPos(p3)); + assertTrue(ml.isNoMessagesAfterPos(p4)); + assertTrue(ml.isNoMessagesAfterPos(p5)); + assertTrue(ml.isNoMessagesAfterPos(p6)); + assertTrue(ml.isNoMessagesAfterPos(PositionImpl.get(p6.getLedgerId(), p6.getEntryId() + 1))); + assertTrue(ml.isNoMessagesAfterPos(PositionImpl.get(p6.getLedgerId() + 1, -1))); + + // cleanup. + ml.close(); + } + @Test public void testGetEstimatedBacklogSize() throws Exception { ManagedLedgerConfig config = new ManagedLedgerConfig(); From 736bd5e65eee77113dac7ee7832275bb8d659933 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 1 Sep 2023 18:02:06 +0800 Subject: [PATCH 259/494] [fix][broker] Fix unsubscribe non-durable subscription error (#21099) --- .../service/persistent/PersistentTopic.java | 11 ++++---- .../broker/service/BrokerServiceTest.java | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 3e3363b8c5b75..948c671044720 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -1161,15 +1161,14 @@ public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { private void asyncDeleteCursorWithClearDelayedMessage(String subscriptionName, CompletableFuture unsubscribeFuture) { - if (!isDelayedDeliveryEnabled() - || !(brokerService.getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory)) { - asyncDeleteCursor(subscriptionName, unsubscribeFuture); - return; - } - PersistentSubscription persistentSubscription = subscriptions.get(subscriptionName); if (persistentSubscription == null) { log.warn("[{}][{}] Can't find subscription, skip clear delayed message", topic, subscriptionName); + unsubscribeFuture.complete(null); + return; + } + if (!isDelayedDeliveryEnabled() + || !(brokerService.getDelayedDeliveryTrackerFactory() instanceof BucketDelayedDeliveryTrackerFactory)) { asyncDeleteCursor(subscriptionName, unsubscribeFuture); return; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 9012c6edb2f70..04018d5fb9d0b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -91,6 +91,7 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; +import org.apache.pulsar.client.api.SubscriptionMode; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.ConnectionPool; @@ -1623,4 +1624,29 @@ public void testDuplicateAcknowledgement() throws Exception { assertEquals(admin.topics().getStats(topicName).getSubscriptions() .get("sub-1").getUnackedMessages(), 0); } + + @Test + public void testUnsubscribeNonDurableSub() throws Exception { + final String ns = "prop/ns-test"; + final String topic = ns + "/testUnsubscribeNonDurableSub"; + + admin.namespaces().createNamespace(ns, 2); + admin.topics().createPartitionedTopic(String.format("persistent://%s", topic), 1); + + pulsarClient.newProducer(Schema.STRING).topic(topic).create().close(); + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.STRING) + .topic(topic) + .subscriptionMode(SubscriptionMode.NonDurable) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscriptionName("sub1") + .subscriptionType(SubscriptionType.Shared) + .subscribe(); + try { + consumer.unsubscribe(); + } catch (Exception ex) { + fail("Unsubscribe failed"); + } + } } From 6d570318eb9370afb7e7f4b6396026d46b2840e9 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Tue, 5 Sep 2023 10:35:49 +0800 Subject: [PATCH 260/494] [fix][fn] Fix the --batch-builder not working error for functions (#21023) --- .../pulsar/functions/utils/FunctionConfigUtils.java | 6 ++++++ .../functions/utils/FunctionConfigUtilsTest.java | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index d20d50df1bdd1..8cbb2ba038809 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -280,6 +280,12 @@ public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFu } sinkSpecBuilder.setProducerSpec(pbldr.build()); } + if (functionConfig.getBatchBuilder() != null) { + Function.ProducerSpec.Builder builder = sinkSpecBuilder.getProducerSpec() != null + ? sinkSpecBuilder.getProducerSpec().toBuilder() + : Function.ProducerSpec.newBuilder(); + sinkSpecBuilder.setProducerSpec(builder.setBatchBuilder(functionConfig.getBatchBuilder()).build()); + } functionDetailsBuilder.setSink(sinkSpecBuilder); if (functionConfig.getTenant() != null) { diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java index 8b4470d8c76c7..4070b1881c3d5 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java @@ -158,6 +158,18 @@ public void testConvertWindow() { ); } + @Test + public void testConvertBatchBuilder() { + FunctionConfig functionConfig = createFunctionConfig(); + functionConfig.setBatchBuilder("KEY_BASED"); + + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + assertEquals(functionDetails.getSink().getProducerSpec().getBatchBuilder(), "KEY_BASED"); + + FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); + assertEquals(convertedConfig.getProducerConfig().getBatchBuilder(), "KEY_BASED"); + } + @Test public void testMergeEqual() { FunctionConfig functionConfig = createFunctionConfig(); From 792a98eedba1c1457651dbe2f9f3afb3a242f1a2 Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:26:52 +0800 Subject: [PATCH 261/494] [fix] [broker] consider iowait as idle. (#19110) --- .../broker/loadbalance/LinuxInfoUtils.java | 5 +- .../impl/LinuxBrokerHostUsageImpl.java | 3 +- .../loadbalance/LinuxInfoUtilsTest.java | 50 +++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtilsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java index 17aa7170fc63c..61f34ef4901ba 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtils.java @@ -161,7 +161,8 @@ public static long getCpuUsageForCGroup() { * *

* Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this - * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal. + * far. Real CPU usage should equal the sum substracting the idle cycles(that is idle+iowait), this would include + * cpu, user, nice, system, irq, softirq, steal, guest and guest_nice. */ public static ResourceUsage getCpuUsageForEntireHost() { try (Stream stream = Files.lines(Paths.get(PROC_STAT_PATH))) { @@ -175,7 +176,7 @@ public static ResourceUsage getCpuUsageForEntireHost() { .filter(s -> !s.contains("cpu")) .mapToLong(Long::parseLong) .sum(); - long idle = Long.parseLong(words[4]); + long idle = Long.parseLong(words[4]) + Long.parseLong(words[5]); return ResourceUsage.builder() .usage(total - idle) .idle(idle) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java index 2f7ca614943b1..6d0e6bb907346 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LinuxBrokerHostUsageImpl.java @@ -155,7 +155,8 @@ private double getTotalCpuUsageForCGroup(double elapsedTimeSeconds) { * * * Line is split in "words", filtering the first. The sum of all numbers give the amount of cpu cycles used this - * far. Real CPU usage should equal the sum subtracting the idle cycles, this would include iowait, irq and steal. + * far. Real CPU usage should equal the sum substracting the idle cycles(that is idle+iowait), this would include + * cpu, user, nice, system, irq, softirq, steal, guest and guest_nice. */ private double getTotalCpuUsageForEntireHost() { LinuxInfoUtils.ResourceUsage cpuUsageForEntireHost = getCpuUsageForEntireHost(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtilsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtilsTest.java new file mode 100644 index 0000000000000..ac21b30bdde51 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LinuxInfoUtilsTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockStatic; +import static org.testng.Assert.assertEquals; +import java.nio.file.Files; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.mockito.MockedStatic; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class LinuxInfoUtilsTest { + + /** + * simulate reading first line of /proc/stat to get total cpu usage. + */ + @Test + public void testGetCpuUsageForEntireHost(){ + try (MockedStatic filesMockedStatic = mockStatic(Files.class)) { + filesMockedStatic.when(() -> Files.lines(any())).thenReturn( + Stream.generate(() -> "cpu 317808 128 58637 2503692 7634 0 13472 0 0 0")); + long idle = 2503692 + 7634, total = 2901371; + LinuxInfoUtils.ResourceUsage resourceUsage = LinuxInfoUtils.ResourceUsage.builder() + .usage(total - idle) + .idle(idle) + .total(total).build(); + assertEquals(LinuxInfoUtils.getCpuUsageForEntireHost(), resourceUsage); + } + } +} From 608f760a17b1b036701cb5cc8ecbb7828863b618 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:58:46 +0800 Subject: [PATCH 262/494] [fix][broker]Fix chunked messages will be filtered by duplicating (#20948) ## Motivation Make the chunk message function work properly when deduplication is enabled. ## Modification ### Only check and store the sequence ID of the last chunk in a chunk message. For example: ```markdown Chunk-1 sequence ID: 0, chunk ID: 0, total chunk: 2 Chunk-2 sequence ID: 0, chunk ID: 1 Chunk-3 sequence ID: 1, chunk ID: 0 total chunk: 3 Chunk-4 sequence ID: 1, chunk ID: 1 Chunk-5 sequence ID: 1, chunk ID: 1 Chunk-6 sequence ID: 1, chunk ID: 2 ``` Only store check and store the sequence ID of Chunk-2 and Chunk-6. **Add a property in the publishContext to determine whether this chunk is the last chunk when persistent completely.** ```java publishContext.setProperty(IS_LAST_CHUNK, Boolean.FALSE); ``` ### Filter and ack duplicated chunks in a chunk message instead of discarding ctx. For example: ```markdown Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 Chunk-3 sequence ID: 0, chunk ID: 2, msgID: 1:3 Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4 Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5 Chunk-6 sequence ID: 0, chunk ID: 3, msgID: 1:6 ``` We should filter and ack chunk-4 and chunk-5. --- .../persistent/MessageDeduplication.java | 34 +++++- .../MessageChunkingDeduplicationTest.java | 114 ++++++++++++++++++ .../impl/MessageChunkingSharedTest.java | 40 ++++-- .../pulsar/client/impl/ConsumerImpl.java | 25 ++++ 4 files changed, 198 insertions(+), 15 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index d1b4b74945f8e..238dc740509b1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -56,6 +56,8 @@ public class MessageDeduplication { private final ManagedLedger managedLedger; private ManagedCursor managedCursor; + private static final String IS_LAST_CHUNK = "isLastChunk"; + enum Status { // Deduplication is initialized @@ -328,11 +330,12 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade String producerName = publishContext.getProducerName(); long sequenceId = publishContext.getSequenceId(); long highestSequenceId = Math.max(publishContext.getHighestSequenceId(), sequenceId); + MessageMetadata md = null; if (producerName.startsWith(replicatorPrefix)) { // Message is coming from replication, we need to use the original producer name and sequence id // for the purpose of deduplication and not rely on the "replicator" name. int readerIndex = headersAndPayload.readerIndex(); - MessageMetadata md = Commands.parseMessageMetadata(headersAndPayload); + md = Commands.parseMessageMetadata(headersAndPayload); producerName = md.getProducerName(); sequenceId = md.getSequenceId(); highestSequenceId = Math.max(md.getHighestSequenceId(), sequenceId); @@ -341,7 +344,23 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade publishContext.setOriginalHighestSequenceId(highestSequenceId); headersAndPayload.readerIndex(readerIndex); } - + long chunkID = -1; + long totalChunk = -1; + if (publishContext.isChunked()) { + if (md == null) { + int readerIndex = headersAndPayload.readerIndex(); + md = Commands.parseMessageMetadata(headersAndPayload); + headersAndPayload.readerIndex(readerIndex); + } + chunkID = md.getChunkId(); + totalChunk = md.getNumChunksFromMsg(); + } + // All chunks of a message use the same message metadata and sequence ID, + // so we only need to check the sequence ID for the last chunk in a chunk message. + if (chunkID != -1 && chunkID != totalChunk - 1) { + publishContext.setProperty(IS_LAST_CHUNK, Boolean.FALSE); + return MessageDupStatus.NotDup; + } // Synchronize the get() and subsequent put() on the map. This would only be relevant if the producer // disconnects and re-connects very quickly. At that point the call can be coming from a different thread synchronized (highestSequencedPushed) { @@ -367,6 +386,11 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade } highestSequencedPushed.put(producerName, highestSequenceId); } + // Only put sequence ID into highestSequencedPushed and + // highestSequencedPersisted until receive and persistent the last chunk. + if (chunkID != -1 && chunkID == totalChunk - 1) { + publishContext.setProperty(IS_LAST_CHUNK, Boolean.TRUE); + } return MessageDupStatus.NotDup; } @@ -387,8 +411,10 @@ public void recordMessagePersisted(PublishContext publishContext, PositionImpl p sequenceId = publishContext.getOriginalSequenceId(); highestSequenceId = publishContext.getOriginalHighestSequenceId(); } - - highestSequencedPersisted.put(producerName, Math.max(highestSequenceId, sequenceId)); + Boolean isLastChunk = (Boolean) publishContext.getProperty(IS_LAST_CHUNK); + if (isLastChunk == null || isLastChunk) { + highestSequencedPersisted.put(producerName, Math.max(highestSequenceId, sequenceId)); + } if (++snapshotCounter >= snapshotInterval) { snapshotCounter = 0; takeSnapshot(position); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java new file mode 100644 index 0000000000000..5e590414132a5 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.apache.pulsar.client.impl.MessageChunkingSharedTest.sendChunk; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-impl") +public class MessageChunkingDeduplicationTest extends ProducerConsumerBase { + + @BeforeClass + @Override + protected void setup() throws Exception { + this.conf.setBrokerDeduplicationEnabled(true); + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testSendChunkMessageWithSameSequenceID() throws Exception { + String topicName = "persistent://my-property/my-ns/testSendChunkMessageWithSameSequenceID"; + String producerName = "test-producer"; + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.STRING) + .subscriptionName("test-sub") + .topic(topicName) + .subscribe(); + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.STRING) + .producerName(producerName) + .topic(topicName) + .enableChunking(true) + .enableBatching(false) + .create(); + int messageSize = 6000; // payload size in KB + String message = "a".repeat(messageSize * 1000); + producer.newMessage().value(message).sequenceId(10).send(); + Message msg = consumer.receive(10, TimeUnit.SECONDS); + assertNotNull(msg); + assertTrue(msg.getMessageId() instanceof ChunkMessageIdImpl); + assertEquals(msg.getValue(), message); + producer.newMessage().value(message).sequenceId(10).send(); + msg = consumer.receive(3, TimeUnit.SECONDS); + assertNull(msg); + } + + @Test + public void testDeduplicateChunksInSingleChunkMessages() throws Exception { + String topicName = "persistent://my-property/my-ns/testDeduplicateChunksInSingleChunkMessage"; + String producerName = "test-producer"; + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.STRING) + .subscriptionName("test-sub") + .topic(topicName) + .subscribe(); + final PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(topicName).get().orElse(null); + assertNotNull(persistentTopic); + sendChunk(persistentTopic, producerName, 1, 0, 2); + sendChunk(persistentTopic, producerName, 1, 1, 2); + sendChunk(persistentTopic, producerName, 1, 1, 2); + + Message message = consumer.receive(15, TimeUnit.SECONDS); + assertEquals(message.getData().length, 2); + + sendChunk(persistentTopic, producerName, 2, 0, 3); + sendChunk(persistentTopic, producerName, 2, 1, 3); + sendChunk(persistentTopic, producerName, 2, 1, 3); + sendChunk(persistentTopic, producerName, 2, 2, 3); + message = consumer.receive(20, TimeUnit.SECONDS); + assertEquals(message.getData().length, 3); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingSharedTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingSharedTest.java index 163c42d835b35..3d24d3746d66a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingSharedTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingSharedTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -34,6 +35,7 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; @@ -45,7 +47,6 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; -import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.protocol.Commands; import org.awaitility.Awaitility; @@ -217,7 +218,7 @@ private static void sendNonChunk(final PersistentTopic persistentTopic, sendChunk(persistentTopic, producerName, sequenceId, null, null); } - private static void sendChunk(final PersistentTopic persistentTopic, + protected static void sendChunk(final PersistentTopic persistentTopic, final String producerName, final long sequenceId, final Integer chunkId, @@ -233,16 +234,33 @@ private static void sendChunk(final PersistentTopic persistentTopic, metadata.setTotalChunkMsgSize(numChunks); } final ByteBuf buf = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata, - PulsarByteBufAllocator.DEFAULT.buffer(1)); - persistentTopic.publishMessage(buf, (e, ledgerId, entryId) -> { - String name = producerName + "-" + sequenceId; - if (chunkId != null) { - name += "-" + chunkId + "-" + numChunks; + Unpooled.wrappedBuffer("a".getBytes())); + persistentTopic.publishMessage(buf, new Topic.PublishContext() { + @Override + public boolean isChunked() { + return chunkId != null; } - if (e == null) { - log.info("Sent {} to ({}, {})", name, ledgerId, entryId); - } else { - log.error("Failed to send {}: {}", name, e.getMessage()); + + @Override + public String getProducerName() { + return producerName; + } + + public long getSequenceId() { + return sequenceId; + } + + @Override + public void completed(Exception e, long ledgerId, long entryId) { + String name = producerName + "-" + sequenceId; + if (chunkId != null) { + name += "-" + chunkId + "-" + numChunks; + } + if (e == null) { + log.info("Sent {} to ({}, {})", name, ledgerId, entryId); + } else { + log.error("Failed to send {}: {}", name, e.getMessage()); + } } }); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 47aecd8052c29..d0b794ad51ea8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.Collections; import java.util.HashMap; @@ -1452,6 +1453,30 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m // discard message if chunk is out-of-order if (chunkedMsgCtx == null || chunkedMsgCtx.chunkedMsgBuffer == null || msgMetadata.getChunkId() != (chunkedMsgCtx.lastChunkedMessageId + 1)) { + // Filter and ack duplicated chunks instead of discard ctx. + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 2, msgID: 1:3 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5 + // Chunk-6 sequence ID: 0, chunk ID: 3, msgID: 1:6 + // We should filter and ack chunk-4 and chunk-5. + if (chunkedMsgCtx != null && msgMetadata.getChunkId() <= chunkedMsgCtx.lastChunkedMessageId) { + log.warn("[{}] Receive a duplicated chunk message with messageId [{}], last-chunk-Id [{}], " + + "chunkId [{}], sequenceId [{}]", + msgMetadata.getProducerName(), msgId, chunkedMsgCtx.lastChunkedMessageId, + msgMetadata.getChunkId(), msgMetadata.getSequenceId()); + compressedPayload.release(); + increaseAvailablePermits(cnx); + boolean repeatedlyReceived = Arrays.stream(chunkedMsgCtx.chunkedMessageIds) + .anyMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId() + && messageId1.entryId == messageId.getEntryId()); + if (!repeatedlyReceived) { + doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null); + } + return null; + } // means we lost the first chunk: should never happen log.info("[{}] [{}] Received unexpected chunk messageId {}, last-chunk-id = {}, chunkId = {}", topic, subscription, msgId, From 946fccf3a3d0106e0992ba05805aca90fe420e06 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Thu, 31 Aug 2023 17:13:09 +0800 Subject: [PATCH 263/494] [fix][fn] Fix ProducerConfig cannot update error (#21037) --- .../functions/utils/FunctionConfigUtils.java | 3 +++ .../functions/utils/SourceConfigUtils.java | 3 +++ .../utils/FunctionConfigUtilsTest.java | 24 +++++++++++++++++++ .../utils/SourceConfigUtilsTest.java | 24 +++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index 8cbb2ba038809..15119050c57be 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -1098,6 +1098,9 @@ public static FunctionConfig validateUpdate(FunctionConfig existingConfig, Funct if (!StringUtils.isEmpty(newConfig.getCustomRuntimeOptions())) { mergedConfig.setCustomRuntimeOptions(newConfig.getCustomRuntimeOptions()); } + if (newConfig.getProducerConfig() != null) { + mergedConfig.setProducerConfig(newConfig.getProducerConfig()); + } return mergedConfig; } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java index ec0c5c444ccba..f3be015d73754 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java @@ -448,6 +448,9 @@ public static SourceConfig validateUpdate(SourceConfig existingConfig, SourceCon validateBatchSourceConfigUpdate(existingConfig.getBatchSourceConfig(), newConfig.getBatchSourceConfig()); mergedConfig.setBatchSourceConfig(newConfig.getBatchSourceConfig()); } + if (newConfig.getProducerConfig() != null) { + mergedConfig.setProducerConfig(newConfig.getProducerConfig()); + } return mergedConfig; } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java index 4070b1881c3d5..8f46199e8ffd5 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java @@ -441,6 +441,30 @@ public void testMergeDifferentWindowConfig() { ); } + @Test + public void testMergeDifferentProducerConfig() { + FunctionConfig functionConfig = createFunctionConfig(); + + ProducerConfig producerConfig = new ProducerConfig(); + producerConfig.setMaxPendingMessages(100); + producerConfig.setMaxPendingMessagesAcrossPartitions(1000); + producerConfig.setUseThreadLocalProducers(true); + producerConfig.setBatchBuilder("DEFAULT"); + producerConfig.setCompressionType(CompressionType.ZLIB); + FunctionConfig newFunctionConfig = createUpdatedFunctionConfig("producerConfig", producerConfig); + + FunctionConfig mergedConfig = FunctionConfigUtils.validateUpdate(functionConfig, newFunctionConfig); + assertEquals( + mergedConfig.getProducerConfig(), + producerConfig + ); + mergedConfig.setProducerConfig(functionConfig.getProducerConfig()); + assertEquals( + new Gson().toJson(functionConfig), + new Gson().toJson(mergedConfig) + ); + } + @Test public void testMergeDifferentTimeout() { FunctionConfig functionConfig = createFunctionConfig(); diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java index 49313dbf02c62..a4da4203d9641 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SourceConfigUtilsTest.java @@ -285,6 +285,30 @@ public void testMergeDifferentBatchSourceConfig() { ); } + @Test + public void testMergeDifferentProducerConfig() { + SourceConfig sourceConfig = createSourceConfig(); + + ProducerConfig producerConfig = new ProducerConfig(); + producerConfig.setMaxPendingMessages(100); + producerConfig.setMaxPendingMessagesAcrossPartitions(1000); + producerConfig.setUseThreadLocalProducers(true); + producerConfig.setBatchBuilder("DEFAULT"); + producerConfig.setCompressionType(CompressionType.ZLIB); + SourceConfig newSourceConfig = createUpdatedSourceConfig("producerConfig", producerConfig); + + SourceConfig mergedConfig = SourceConfigUtils.validateUpdate(sourceConfig, newSourceConfig); + assertEquals( + mergedConfig.getProducerConfig(), + producerConfig + ); + mergedConfig.setProducerConfig(sourceConfig.getProducerConfig()); + assertEquals( + new Gson().toJson(sourceConfig), + new Gson().toJson(mergedConfig) + ); + } + @Test public void testValidateConfig() { SourceConfig sourceConfig = createSourceConfig(); From 937fa03d8fd8da8664f1cc29bb847d524aec9342 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Thu, 31 Aug 2023 16:19:58 +0800 Subject: [PATCH 264/494] [fix][io] Fix --retain[-key]-ordering not working error for sink (#21060) --- .../org/apache/pulsar/functions/utils/SinkConfigUtils.java | 7 +++++++ .../apache/pulsar/functions/utils/SinkConfigUtilsTest.java | 2 ++ 2 files changed, 9 insertions(+) diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index d79f787588c95..7f974bb4b1945 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -216,6 +216,13 @@ public static FunctionDetails convert(SinkConfig sinkConfig, ExtractedSinkDetail functionDetailsBuilder.setSource(sourceSpecBuilder); + if (sinkConfig.getRetainKeyOrdering() != null) { + functionDetailsBuilder.setRetainKeyOrdering(sinkConfig.getRetainKeyOrdering()); + } + if (sinkConfig.getRetainOrdering() != null) { + functionDetailsBuilder.setRetainOrdering(sinkConfig.getRetainOrdering()); + } + if (sinkConfig.getMaxMessageRetries() != null && sinkConfig.getMaxMessageRetries() > 0) { Function.RetryDetails.Builder retryDetails = Function.RetryDetails.newBuilder(); retryDetails.setMaxMessageRetries(sinkConfig.getMaxMessageRetries()); diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java index 62d6f68d31b2c..8ac9b61e3f60f 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java @@ -151,6 +151,7 @@ public void testParseRetainOrderingField() throws IOException { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setRetainOrdering(testcase); Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); + assertEquals(functionDetails.getRetainOrdering(), testcase != null ? testcase : Boolean.valueOf(false)); SinkConfig result = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals(result.getRetainOrdering(), testcase != null ? testcase : Boolean.valueOf(false)); } @@ -163,6 +164,7 @@ public void testParseKeyRetainOrderingField() throws IOException { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setRetainKeyOrdering(testcase); Function.FunctionDetails functionDetails = SinkConfigUtils.convert(sinkConfig, new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); + assertEquals(functionDetails.getRetainKeyOrdering(), testcase != null ? testcase : Boolean.valueOf(false)); SinkConfig result = SinkConfigUtils.convertFromDetails(functionDetails); assertEquals(result.getRetainKeyOrdering(), testcase != null ? testcase : Boolean.valueOf(false)); } From e2812bf99cc7daaa19391eec1b02a19cc1b3bcbd Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:38:11 +0800 Subject: [PATCH 265/494] [fix] [broker] remove bundle-data in local metadata store. (#21078) Motivation: When deleting a namespace, we will delete znode under the path `/loadbalance/bundle-data` in `local metadata store` instead of `global metadata store`. Modifications: Delete bundle data znode in local metadata store. --- .../broker/resources/NamespaceResources.java | 8 ++-- .../broker/resources/PulsarResources.java | 3 +- .../resources/NamespaceResourcesTest.java | 44 +++++++++++++++++++ .../pulsar/broker/admin/AdminApi2Test.java | 31 ++++++++++--- .../broker/testcontext/PulsarTestContext.java | 3 +- 5 files changed, 79 insertions(+), 10 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index 99ce288fa012c..e35c208c208db 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -49,16 +49,18 @@ public class NamespaceResources extends BaseResources { private final IsolationPolicyResources isolationPolicies; private final PartitionedTopicResources partitionedTopicResources; private final MetadataStore configurationStore; + private final MetadataStore localStore; public static final String POLICIES_READONLY_FLAG_PATH = "/admin/flags/policies-readonly"; private static final String NAMESPACE_BASE_PATH = "/namespace"; private static final String BUNDLE_DATA_BASE_PATH = "/loadbalance/bundle-data"; - public NamespaceResources(MetadataStore configurationStore, int operationTimeoutSec) { + public NamespaceResources(MetadataStore localStore, MetadataStore configurationStore, int operationTimeoutSec) { super(configurationStore, Policies.class, operationTimeoutSec); this.configurationStore = configurationStore; isolationPolicies = new IsolationPolicyResources(configurationStore, operationTimeoutSec); partitionedTopicResources = new PartitionedTopicResources(configurationStore, operationTimeoutSec); + this.localStore = localStore; } public CompletableFuture> listNamespacesAsync(String tenant) { @@ -381,13 +383,13 @@ public CompletableFuture runWithMarkDeleteAsync(TopicName topic, // clear resource of `/loadbalance/bundle-data/{tenant}/{namespace}/` in metadata-store public CompletableFuture deleteBundleDataAsync(NamespaceName ns) { final String namespaceBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, ns.toString()); - return getStore().deleteRecursive(namespaceBundlePath); + return this.localStore.deleteRecursive(namespaceBundlePath); } // clear resource of `/loadbalance/bundle-data/{tenant}/` in metadata-store public CompletableFuture deleteBundleDataTenantAsync(String tenant) { final String tenantBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, tenant); - return getStore().deleteRecursive(tenantBundlePath); + return this.localStore.deleteRecursive(tenantBundlePath); } } \ No newline at end of file diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java index dfcd0a4194ff5..a3c5633a6dbe8 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/PulsarResources.java @@ -60,7 +60,8 @@ public PulsarResources(MetadataStore localMetadataStore, MetadataStore configura if (configurationMetadataStore != null) { tenantResources = new TenantResources(configurationMetadataStore, operationTimeoutSec); clusterResources = new ClusterResources(configurationMetadataStore, operationTimeoutSec); - namespaceResources = new NamespaceResources(configurationMetadataStore, operationTimeoutSec); + namespaceResources = new NamespaceResources(localMetadataStore, configurationMetadataStore + , operationTimeoutSec); resourcegroupResources = new ResourceGroupResources(configurationMetadataStore, operationTimeoutSec); } else { tenantResources = null; diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java index 85f54a76dc3c4..deb86e1802f6f 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/resources/NamespaceResourcesTest.java @@ -18,12 +18,34 @@ */ package org.apache.pulsar.broker.resources; +import static org.apache.pulsar.broker.resources.BaseResources.joinPath; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; + +import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.metadata.api.MetadataStore; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; public class NamespaceResourcesTest { + + private MetadataStore localStore; + private MetadataStore configurationStore; + private NamespaceResources namespaceResources; + + private static final String BUNDLE_DATA_BASE_PATH = "/loadbalance/bundle-data"; + + @BeforeMethod + public void setup() { + localStore = mock(MetadataStore.class); + configurationStore = mock(MetadataStore.class); + namespaceResources = new NamespaceResources(localStore, configurationStore, 30); + } + @Test public void test_pathIsFromNamespace() { assertFalse(NamespaceResources.pathIsFromNamespace("/admin/clusters")); @@ -31,4 +53,26 @@ public void test_pathIsFromNamespace() { assertFalse(NamespaceResources.pathIsFromNamespace("/admin/policies/my-tenant")); assertTrue(NamespaceResources.pathIsFromNamespace("/admin/policies/my-tenant/my-ns")); } + + /** + * Test that the bundle-data node is deleted from the local stores. + */ + @Test + public void testDeleteBundleDataAsync() { + NamespaceName ns = NamespaceName.get("my-tenant/my-ns"); + String namespaceBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, ns.toString()); + namespaceResources.deleteBundleDataAsync(ns); + + String tenant="my-tenant"; + String tenantBundlePath = joinPath(BUNDLE_DATA_BASE_PATH, tenant); + namespaceResources.deleteBundleDataTenantAsync(tenant); + + verify(localStore).deleteRecursive(namespaceBundlePath); + verify(localStore).deleteRecursive(tenantBundlePath); + + assertThrows(()-> verify(configurationStore).deleteRecursive(namespaceBundlePath)); + assertThrows(()-> verify(configurationStore).deleteRecursive(tenantBundlePath)); + } + + } \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 8a5b4531dbe5e..b4fcd21df3704 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -67,6 +67,7 @@ import org.apache.pulsar.broker.admin.AdminApiTest.MockedPulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; +import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; @@ -1720,6 +1721,8 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { // Set conf. cleanup(); setNamespaceAttr(namespaceAttr); + this.conf.setMetadataStoreUrl("127.0.0.1:2181"); + this.conf.setConfigurationMetadataStoreUrl("127.0.0.1:2182"); setup(); String tenant = newUniqueName("test-tenant"); @@ -1740,6 +1743,28 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { admin.topics().createPartitionedTopic(topic, 10); assertFalse(admin.topics().getList(namespace).isEmpty()); + final String managedLedgersPath = "/managed-ledgers/" + namespace; + final String bundleDataPath = "/loadbalance/bundle-data/" + namespace; + // Trigger bundle owned by brokers. + pulsarClient.newProducer().topic(topic).create().close(); + // Trigger bundle data write to ZK. + Awaitility.await().untilAsserted(() -> { + boolean bundleDataWereWriten = false; + for (PulsarService ps : new PulsarService[]{pulsar, mockPulsarSetup.getPulsar()}) { + ModularLoadManagerWrapper loadManager = (ModularLoadManagerWrapper) ps.getLoadManager().get(); + ModularLoadManagerImpl loadManagerImpl = (ModularLoadManagerImpl) loadManager.getLoadManager(); + ps.getBrokerService().updateRates(); + loadManagerImpl.updateLocalBrokerData(); + loadManagerImpl.writeBundleDataOnZooKeeper(); + bundleDataWereWriten = bundleDataWereWriten || ps.getLocalMetadataStore().exists(bundleDataPath).join(); + } + assertTrue(bundleDataWereWriten); + }); + + // assert znode exists in metadata store + assertTrue(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); + assertTrue(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); + try { admin.namespaces().deleteNamespace(namespace, false); fail("should have failed due to namespace not empty"); @@ -1756,12 +1781,8 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception { assertFalse(admin.namespaces().getNamespaces(tenant).contains(namespace)); assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty()); - - final String managedLedgersPath = "/managed-ledgers/" + namespace; + // assert znode deleted in metadata store assertFalse(pulsar.getLocalMetadataStore().exists(managedLedgersPath).join()); - - - final String bundleDataPath = "/loadbalance/bundle-data/" + namespace; assertFalse(pulsar.getLocalMetadataStore().exists(bundleDataPath).join()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java index 062047c7133f7..379b5cf63fffd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/PulsarTestContext.java @@ -699,7 +699,8 @@ protected void initializePulsarServices(SpyConfig spyConfig, Builder builder) { if (metadataStore == null) { metadataStore = builder.configurationMetadataStore; } - NamespaceResources nsr = spyConfigPulsarResources.spy(NamespaceResources.class, metadataStore, 30); + NamespaceResources nsr = spyConfigPulsarResources.spy(NamespaceResources.class, + builder.localMetadataStore, metadataStore, 30); TopicResources tsr = spyConfigPulsarResources.spy(TopicResources.class, metadataStore); pulsarResources( spyConfigPulsarResources.spy( From 9c6671eafd81d76aa7327bd7cab24a375b33a644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= Date: Thu, 31 Aug 2023 16:18:17 +0800 Subject: [PATCH 266/494] [fix][client] Fix logging problem in pulsar client (#21094) --- .../java/org/apache/pulsar/client/impl/ConsumerImpl.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index d0b794ad51ea8..8263e7f3198f5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -413,7 +413,7 @@ public CompletableFuture unsubscribeAsync() { unsubscribeFuture.completeExceptionally( PulsarClientException.wrap(e.getCause(), String.format("Failed to unsubscribe the subscription %s of topic %s", - topicName.toString(), subscription))); + subscription, topicName.toString()))); return null; }); } else { @@ -2485,9 +2485,9 @@ private void internalGetLastMessageIdAsync(final Backoff backoff, return; } + log.warn("[{}] [{}] Could not get connection while getLastMessageId -- Will try again in {} ms", + topic, getHandlerName(), nextDelay); ((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor()).schedule(() -> { - log.warn("[{}] [{}] Could not get connection while getLastMessageId -- Will try again in {} ms", - topic, getHandlerName(), nextDelay); remainingTime.addAndGet(-nextDelay); internalGetLastMessageIdAsync(backoff, remainingTime, future); }, nextDelay, TimeUnit.MILLISECONDS); From f3b50d6c114f12ec8e65fa0234f8ab796ed5cf33 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 4 Sep 2023 08:50:49 +0800 Subject: [PATCH 267/494] [fix][client] Avoid ack hole for chunk message (#21101) ## Motivation Handle ack hole case: For example: ```markdown Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:3 Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4 Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5 ``` Consumer ack chunk message via ChunkMessageIdImpl that consists of all the chunks in this chunk message(Chunk-3, Chunk-4, Chunk-5). The Chunk-1 and Chunk-2 are not included in the ChunkMessageIdImpl, so we should process it here. ## Modification Ack chunk-1 and chunk-2. --- .../client/impl/MessageChunkingTest.java | 33 +++++++++++++ .../pulsar/client/impl/ConsumerImpl.java | 46 ++++++++++++++++--- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java index dffa003524864..f266afd8a2ee1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java @@ -356,6 +356,38 @@ public void testMaxPendingChunkMessages() throws Exception { assertNull(consumer.receive(5, TimeUnit.SECONDS)); } + @Test + public void testResendChunkMessagesWithoutAckHole() throws Exception { + log.info("-- Starting {} test --", methodName); + final String topicName = "persistent://my-property/my-ns/testResendChunkMessagesWithoutAckHole"; + final String subName = "my-subscriber-name"; + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .maxPendingChunkedMessage(10) + .autoAckOldestChunkedMessageOnQueueFull(true) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .chunkMaxMessageSize(100) + .enableChunking(true) + .enableBatching(false) + .create(); + + sendSingleChunk(producer, "0", 0, 2); + + sendSingleChunk(producer, "0", 0, 2); // Resending the first chunk + sendSingleChunk(producer, "0", 1, 2); + + Message receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-0-0|chunk-0-1|"); + consumer.acknowledge(receivedMsg); + assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName) + .getNonContiguousDeletedMessagesRanges(), 0); + } + @Test public void testResendChunkMessages() throws Exception { log.info("-- Starting {} test --", methodName); @@ -395,6 +427,7 @@ public void testResendChunkMessages() throws Exception { receivedMsg = consumer.receive(5, TimeUnit.SECONDS); assertEquals(receivedMsg.getValue(), "chunk-1-0|chunk-1-1|chunk-1-2|"); consumer.acknowledge(receivedMsg); + Assert.assertEquals(((ConsumerImpl) consumer).getAvailablePermits(), 8); } /** diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 8263e7f3198f5..3a542e6610419 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1413,7 +1413,9 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId, MessageIdData messageId, ClientCnx cnx) { - + if (msgMetadata.getChunkId() != (msgMetadata.getNumChunksFromMsg() - 1)) { + increaseAvailablePermits(cnx); + } // Lazy task scheduling to expire incomplete chunk message if (expireTimeOfIncompleteChunkedMessageMillis > 0 && expireChunkMessageTaskScheduled.compareAndSet(false, true)) { @@ -1429,6 +1431,37 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m if (msgMetadata.getChunkId() == 0) { if (chunkedMsgCtx != null) { + // Handle ack hole case when receive duplicated chunks. + // There are two situation that receives chunks with the same sequence ID and chunk ID. + // Situation 1 - Message redeliver: + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:3 + // In this case, chunk-3 and chunk-4 have the same msgID with chunk-1 and chunk-2. + // This may be caused by message redeliver, we can't ack any chunk in this case here. + // Situation 2 - Corrupted chunk message + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:3 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5 + // In this case, all the chunks with different msgIDs and are persistent in the topic. + // But Chunk-1 and Chunk-2 belong to a corrupted chunk message that must be skipped since + // they will not be delivered to end users. So we should ack them here to avoid ack hole. + boolean isCorruptedChunkMessageDetected = Arrays.stream(chunkedMsgCtx.chunkedMessageIds) + .noneMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId() + && messageId1.entryId == messageId.getEntryId()); + if (isCorruptedChunkMessageDetected) { + Arrays.stream(chunkedMsgCtx.chunkedMessageIds).forEach(messageId1 -> { + if (messageId1 != null) { + doAcknowledge(messageId1, AckType.Individual, Collections.emptyMap(), null); + } + }); + } // The first chunk of a new chunked-message received before receiving other chunks of previous // chunked-message // so, remove previous chunked-message from map and release buffer @@ -1468,11 +1501,12 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m msgMetadata.getProducerName(), msgId, chunkedMsgCtx.lastChunkedMessageId, msgMetadata.getChunkId(), msgMetadata.getSequenceId()); compressedPayload.release(); - increaseAvailablePermits(cnx); - boolean repeatedlyReceived = Arrays.stream(chunkedMsgCtx.chunkedMessageIds) - .anyMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId() + // Just like the above logic of receiving the first chunk again. We only ack this chunk in the message + // duplication case. + boolean isDuplicatedChunk = Arrays.stream(chunkedMsgCtx.chunkedMessageIds) + .noneMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId() && messageId1.entryId == messageId.getEntryId()); - if (!repeatedlyReceived) { + if (isDuplicatedChunk) { doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null); } return null; @@ -1489,7 +1523,6 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m } chunkedMessagesMap.remove(msgMetadata.getUuid()); compressedPayload.release(); - increaseAvailablePermits(cnx); if (expireTimeOfIncompleteChunkedMessageMillis > 0 && System.currentTimeMillis() > (msgMetadata.getPublishTime() + expireTimeOfIncompleteChunkedMessageMillis)) { @@ -1508,7 +1541,6 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m // if final chunk is not received yet then release payload and return if (msgMetadata.getChunkId() != (msgMetadata.getNumChunksFromMsg() - 1)) { compressedPayload.release(); - increaseAvailablePermits(cnx); return null; } From 115f7137630de9fd9e0d490487633fb734beec49 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Fri, 1 Sep 2023 18:07:46 +0800 Subject: [PATCH 268/494] [fix][broker] Fix deleting topic not delete the related topic policy and schema. (#21093) Fixes #21075 ### Motivation When the topic is loaded, it will delete the topic-level policy if it is enabled. But if the topic is not loaded, it will directly delete through managed ledger factory. But then we will leave the topic policy there. When the topic is created next time, it will use the old topic policy ### Modifications When deleting the topic, delete the schema and topic policies even if the topic is not loaded. --- .../pulsar/broker/service/AbstractTopic.java | 17 +----- .../pulsar/broker/service/BrokerService.java | 55 ++++++++++++++----- .../service/BrokerBkEnsemblesTests.java | 32 +++++++++-- .../systopic/PartitionedSystemTopicTest.java | 25 +++++++++ 4 files changed, 94 insertions(+), 35 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 8e17a02276286..97966828fb117 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -58,7 +58,6 @@ import org.apache.pulsar.broker.service.BrokerServiceException.TopicMigratedException; import org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException; import org.apache.pulsar.broker.service.plugin.EntryFilter; -import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; @@ -674,21 +673,7 @@ private boolean allowAutoUpdateSchema() { @Override public CompletableFuture deleteSchema() { - String id = getSchemaId(); - SchemaRegistryService schemaRegistryService = brokerService.pulsar().getSchemaRegistryService(); - return BookkeeperSchemaStorage.ignoreUnrecoverableBKException(schemaRegistryService.getSchema(id)) - .thenCompose(schema -> { - if (schema != null) { - // It's different from `SchemasResource.deleteSchema` - // because when we delete a topic, the schema - // history is meaningless. But when we delete a schema of a topic, a new schema could be - // registered in the future. - log.info("Delete schema storage of id: {}", id); - return schemaRegistryService.deleteSchemaStorage(id); - } else { - return CompletableFuture.completedFuture(null); - } - }); + return brokerService.deleteSchema(TopicName.get(getName())); } @Override diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 2da3e73d8b68f..d6b17f4faa4da 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -119,6 +119,8 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.SystemTopic; import org.apache.pulsar.broker.service.plugin.EntryFilterProvider; +import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; +import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.prometheus.metrics.ObserverGauge; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; @@ -159,6 +161,7 @@ import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.policies.data.stats.TopicStatsImpl; +import org.apache.pulsar.common.protocol.schema.SchemaVersion; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FieldParser; import org.apache.pulsar.common.util.FutureUtil; @@ -1156,26 +1159,33 @@ private CompletableFuture deleteTopicInternal(String topic, boolean forceD CompletableFuture future = new CompletableFuture<>(); CompletableFuture deleteTopicAuthenticationFuture = new CompletableFuture<>(); deleteTopicAuthenticationWithRetry(topic, deleteTopicAuthenticationFuture, 5); - - deleteTopicAuthenticationFuture.whenComplete((v, ex) -> { + deleteTopicAuthenticationFuture + .thenCompose(__ -> deleteSchema(tn)) + .thenCompose(__ -> { + if (!SystemTopicNames.isTopicPoliciesSystemTopic(topic) + && getPulsar().getConfiguration().isSystemTopicEnabled()) { + return deleteTopicPolicies(tn); + } + return CompletableFuture.completedFuture(null); + }).whenComplete((v, ex) -> { if (ex != null) { future.completeExceptionally(ex); return; } CompletableFuture mlConfigFuture = getManagedLedgerConfig(topicName); managedLedgerFactory.asyncDelete(tn.getPersistenceNamingEncoding(), - mlConfigFuture, new DeleteLedgerCallback() { - @Override - public void deleteLedgerComplete(Object ctx) { - future.complete(null); - } + mlConfigFuture, new DeleteLedgerCallback() { + @Override + public void deleteLedgerComplete(Object ctx) { + future.complete(null); + } - @Override - public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { - future.completeExceptionally(exception); - } - }, null); - }); + @Override + public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) { + future.completeExceptionally(exception); + } + }, null); + }); return future; } @@ -3457,6 +3467,25 @@ public CompletableFuture deleteTopicPolicies(TopicName topicName) { .deleteTopicPoliciesAsync(TopicName.get(topicName.getPartitionedTopicName())); } + CompletableFuture deleteSchema(TopicName topicName) { + String base = topicName.getPartitionedTopicName(); + String id = TopicName.get(base).getSchemaName(); + SchemaRegistryService schemaRegistryService = getPulsar().getSchemaRegistryService(); + return BookkeeperSchemaStorage.ignoreUnrecoverableBKException(schemaRegistryService.getSchema(id)) + .thenCompose(schema -> { + if (schema != null) { + // It's different from `SchemasResource.deleteSchema` + // because when we delete a topic, the schema + // history is meaningless. But when we delete a schema of a topic, a new schema could be + // registered in the future. + log.info("Delete schema storage of id: {}", id); + return getPulsar().getSchemaRegistryService().deleteSchemaStorage(id); + } else { + return CompletableFuture.completedFuture(null); + } + }); + } + private CompletableFuture checkMaxTopicsPerNamespace(TopicName topicName, int numPartitions) { return pulsar.getPulsarResources().getNamespaceResources() .getPoliciesAsync(topicName.getNamespaceObject()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java index 9f19bda3647f3..40649a4164047 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBkEnsemblesTests.java @@ -21,8 +21,8 @@ import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertThrows; import static org.testng.Assert.fail; - import java.lang.reflect.Field; import java.util.Map.Entry; import java.util.NavigableMap; @@ -31,10 +31,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; - import com.google.common.collect.Sets; import lombok.Cleanup; - import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; @@ -47,6 +45,7 @@ import org.apache.bookkeeper.util.StringUtils; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.Producer; @@ -497,10 +496,31 @@ public void testDeleteTopicWithMissingData() throws Exception { // Expected } - // Deletion must succeed - admin.topics().delete(topic); + assertThrows(PulsarAdminException.ServerSideErrorException.class, () -> admin.topics().delete(topic)); + } + + @Test + public void testDeleteTopicWithoutTopicLoaded() throws Exception { + String namespace = BrokerTestUtil.newUniqueName("prop/usc"); + admin.namespaces().createNamespace(namespace); + + String topic = BrokerTestUtil.newUniqueName(namespace + "/my-topic"); + + @Cleanup + PulsarClient client = PulsarClient.builder() + .serviceUrl(pulsar.getBrokerServiceUrl()) + .statsInterval(0, TimeUnit.SECONDS) + .build(); - // Topic will not be there after + @Cleanup + Producer producer = client.newProducer(Schema.STRING) + .topic(topic) + .create(); + + producer.close(); + admin.topics().unload(topic); + + admin.topics().delete(topic); assertEquals(pulsar.getBrokerService().getTopicIfExists(topic).join(), Optional.empty()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index 008c2143a3566..34e7dfe92e5d7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.systopic; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; import com.google.common.collect.Sets; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -37,6 +39,7 @@ import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.schema.SchemaRegistry; import org.apache.pulsar.client.admin.ListTopicsOptions; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; @@ -55,6 +58,7 @@ import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; @@ -299,4 +303,25 @@ public void testSystemTopicNotCheckExceed() throws Exception { writer1.get().close(); writer2.get().close(); } + + @Test + public void testDeleteTopicSchemaAndPolicyWhenTopicIsNotLoaded() throws Exception { + final String ns = "prop/ns-test"; + admin.namespaces().createNamespace(ns, 2); + final String topicName = "persistent://prop/ns-test/testDeleteTopicSchemaAndPolicyWhenTopicIsNotLoaded"; + admin.topics().createNonPartitionedTopic(topicName); + pulsarClient.newProducer(Schema.STRING).topic(topicName).create().close(); + admin.topicPolicies().setMaxConsumers(topicName, 2); + Awaitility.await().untilAsserted(() -> assertEquals(admin.topicPolicies().getMaxConsumers(topicName), 2)); + CompletableFuture> topic = pulsar.getBrokerService().getTopic(topicName, false); + PersistentTopic persistentTopic = (PersistentTopic) topic.join().get(); + persistentTopic.close(); + admin.topics().delete(topicName); + TopicPolicies topicPolicies = pulsar.getTopicPoliciesService().getTopicPoliciesIfExists(TopicName.get(topicName)); + assertNull(topicPolicies); + String base = TopicName.get(topicName).getPartitionedTopicName(); + String id = TopicName.get(base).getSchemaName(); + CompletableFuture schema = pulsar.getSchemaRegistryService().getSchema(id); + assertNull(schema.join()); + } } From 4a97b77f27e33a1fe78dd3d5e614eb3b20e910dc Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 5 Sep 2023 10:28:17 +0800 Subject: [PATCH 269/494] [fix][client] Fix repeat consume when using n-ack and batched messages (#21116) --- .../BatchMessageWithBatchIndexLevelTest.java | 85 +++++++++++++++++++ ...sistentAcknowledgmentsGroupingTracker.java | 13 ++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java index 433f5e56d952d..d04647e21c1be 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -27,19 +27,26 @@ import java.util.concurrent.TimeUnit; import lombok.Cleanup; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +@Slf4j @Test(groups = "broker") public class BatchMessageWithBatchIndexLevelTest extends BatchMessageTest { @@ -280,4 +287,82 @@ public void testAckMessageWithNotOwnerConsumerUnAckMessageCount() throws Excepti Awaitility.await().until(() -> getPulsar().getBrokerService().getTopic(topicName, false) .get().get().getSubscription(subName).getConsumers().get(0).getUnackedMessages() == 0); } + + @Test + public void testNegativeAckAndLongAckDelayWillNotLeadRepeatConsume() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/tp_"); + final String subscriptionName = "s1"; + final int redeliveryDelaySeconds = 2; + + // Create producer and consumer. + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(true) + .batchingMaxMessages(1000) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + ConsumerImpl consumer = (ConsumerImpl) pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subscriptionName) + .subscriptionType(SubscriptionType.Shared) + .negativeAckRedeliveryDelay(redeliveryDelaySeconds, TimeUnit.SECONDS) + .enableBatchIndexAcknowledgment(true) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .acknowledgmentGroupTime(1, TimeUnit.HOURS) + .subscribe(); + + // Send 10 messages in batch. + ArrayList messagesSent = new ArrayList<>(); + List> sendTasks = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + String msg = Integer.valueOf(i).toString(); + sendTasks.add(producer.sendAsync(Integer.valueOf(i).toString())); + messagesSent.add(msg); + } + producer.flush(); + FutureUtil.waitForAll(sendTasks).join(); + + // Receive messages. + ArrayList messagesReceived = new ArrayList<>(); + // NegativeAck "batchMessageIdIndex1" once. + boolean index1HasBeenNegativeAcked = false; + while (true) { + Message message = consumer.receive(2, TimeUnit.SECONDS); + if (message == null) { + break; + } + if (index1HasBeenNegativeAcked) { + messagesReceived.add(message.getValue()); + consumer.acknowledge(message); + continue; + } + if (((MessageIdAdv) message.getMessageId()).getBatchIndex() == 1) { + consumer.negativeAcknowledge(message); + index1HasBeenNegativeAcked = true; + continue; + } + messagesReceived.add(message.getValue()); + consumer.acknowledge(message); + } + + // Receive negative acked messages. + // Wait the message negative acknowledgment finished. + int tripleRedeliveryDelaySeconds = redeliveryDelaySeconds * 3; + while (true) { + Message message = consumer.receive(tripleRedeliveryDelaySeconds, TimeUnit.SECONDS); + if (message == null) { + break; + } + messagesReceived.add(message.getValue()); + consumer.acknowledge(message); + } + + log.info("messagesSent: {}, messagesReceived: {}", messagesSent, messagesReceived); + Assert.assertEquals(messagesReceived.size(), messagesSent.size()); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(topicName); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java index 9086ccc4ef0e0..0cf776aea5942 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PersistentAcknowledgmentsGroupingTracker.java @@ -124,7 +124,18 @@ public boolean isDuplicate(MessageId messageId) { // Already included in a cumulative ack return true; } else { - return pendingIndividualAcks.contains(MessageIdAdvUtils.discardBatch(messageIdAdv)); + // If "batchIndexAckEnabled" is false, the batched messages acknowledgment will be traced by + // pendingIndividualAcks. So no matter what type the message ID is, check with "pendingIndividualAcks" + // first. + MessageIdAdv key = MessageIdAdvUtils.discardBatch(messageIdAdv); + if (pendingIndividualAcks.contains(key)) { + return true; + } + if (messageIdAdv.getBatchIndex() >= 0) { + ConcurrentBitSetRecyclable bitSet = pendingIndividualBatchIndexAcks.get(key); + return bitSet != null && !bitSet.get(messageIdAdv.getBatchIndex()); + } + return false; } } From acb5a54f3f194b7a07062100670e94c0330e2a21 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Mon, 4 Sep 2023 17:49:35 +0800 Subject: [PATCH 270/494] [fix][broker] Cleanup correctly heartbeat bundle ownership when handling broker deletion event (#21083) --- .../pulsar/broker/admin/impl/BrokersBase.java | 5 +- .../channel/ServiceUnitStateChannelImpl.java | 10 ++- .../broker/namespace/NamespaceService.java | 38 ++++----- .../pulsar/broker/admin/AdminApiTest.java | 2 +- .../broker/admin/v1/V1_AdminApiTest.java | 2 +- .../ExtensibleLoadManagerImplTest.java | 8 +- .../channel/ServiceUnitStateChannelTest.java | 81 ++++++++++++++----- .../namespace/NamespaceServiceTest.java | 2 +- .../broker/service/BrokerServiceTest.java | 6 +- .../service/InactiveTopicDeleteTest.java | 6 +- .../systopic/PartitionedSystemTopicTest.java | 6 +- .../pulsar/compaction/CompactionTest.java | 4 +- 12 files changed, 106 insertions(+), 64 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index a61a2c940b866..d37a9eda1abc8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -397,9 +397,10 @@ private void checkDeadlockedThreads() { private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion) { + String lookupServiceAddress = pulsar().getLookupServiceAddress(); NamespaceName namespaceName = (topicVersion == TopicVersion.V2) - ? NamespaceService.getHeartbeatNamespaceV2(pulsar().getAdvertisedAddress(), pulsar().getConfiguration()) - : NamespaceService.getHeartbeatNamespace(pulsar().getAdvertisedAddress(), pulsar().getConfiguration()); + ? NamespaceService.getHeartbeatNamespaceV2(lookupServiceAddress, pulsar().getConfiguration()) + : NamespaceService.getHeartbeatNamespace(lookupServiceAddress, pulsar().getConfiguration()); final String topicName = String.format("persistent://%s/%s", namespaceName, HEALTH_CHECK_TOPIC_SUFFIX); LOG.info("[{}] Running healthCheck with topic={}", clientAppId(), topicName); final String messageStr = UUID.randomUUID().toString(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 4eb25848fda94..e9fbf625edd7b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -41,6 +41,8 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Stable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; +import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_FMT; +import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_FMT_V2; import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; import static org.apache.pulsar.common.topics.TopicCompactionStrategy.TABLE_VIEW_TAG; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; @@ -92,6 +94,7 @@ import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceBundles; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.Metrics; @@ -1213,10 +1216,9 @@ private synchronized void doCleanup(String broker) { int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); String heartbeatNamespace = - NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()) - .toString(); - String heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), - pulsar.getConfiguration()).toString(); + NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT, config.getClusterName(), broker)).toString(); + String heartbeatNamespaceV2 = + NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT_V2, broker)).toString(); Map orphanSystemServiceUnits = new HashMap<>(); for (var etr : tableview.entrySet()) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 141e48515b29b..abf963614089d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -132,8 +132,8 @@ public class NamespaceService implements AutoCloseable { public static final Pattern HEARTBEAT_NAMESPACE_PATTERN = Pattern.compile("pulsar/[^/]+/([^:]+:\\d+)"); public static final Pattern HEARTBEAT_NAMESPACE_PATTERN_V2 = Pattern.compile("pulsar/([^:]+:\\d+)"); public static final Pattern SLA_NAMESPACE_PATTERN = Pattern.compile(SLA_NAMESPACE_PROPERTY + "/[^/]+/([^:]+:\\d+)"); - public static final String HEARTBEAT_NAMESPACE_FMT = "pulsar/%s/%s:%s"; - public static final String HEARTBEAT_NAMESPACE_FMT_V2 = "pulsar/%s:%s"; + public static final String HEARTBEAT_NAMESPACE_FMT = "pulsar/%s/%s"; + public static final String HEARTBEAT_NAMESPACE_FMT_V2 = "pulsar/%s"; public static final String SLA_NAMESPACE_FMT = SLA_NAMESPACE_PROPERTY + "/%s/%s:%s"; private final ConcurrentOpenHashMap namespaceClients; @@ -159,7 +159,7 @@ public class NamespaceService implements AutoCloseable { */ public NamespaceService(PulsarService pulsar) { this.pulsar = pulsar; - host = pulsar.getAdvertisedAddress(); + this.host = pulsar.getAdvertisedAddress(); this.config = pulsar.getConfiguration(); this.loadManager = pulsar.getLoadManager(); this.bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32()); @@ -327,15 +327,17 @@ private CompletableFuture> internalGetWebServiceUrl(Optional optionalTopic = pulsar.getBrokerService() @@ -195,7 +195,7 @@ public void testSystemNamespaceNotCreateChangeEventsTopic() throws Exception { @Test public void testHeartbeatTopicNotAllowedToSendEvent() throws Exception { admin.brokers().healthcheck(TopicVersion.V2); - NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), pulsar.getConfig()); TopicName topicName = TopicName.get("persistent", namespaceName, SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); for (int partition = 0; partition < PARTITIONS; partition ++) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index c5dbd9c49aac9..e603b3ccc4d12 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -1771,9 +1771,9 @@ public void testReadUnCompacted(boolean batchEnabled) throws PulsarClientExcepti @SneakyThrows @Test public void testHealthCheckTopicNotCompacted() { - NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()); + NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getLookupServiceAddress(), pulsar.getConfiguration()); String topicV1 = "persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck"; - NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()); + NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), pulsar.getConfiguration()); String topicV2 = heartbeatNamespaceV2.toString() + "/healthcheck"; Producer producer1 = pulsarClient.newProducer().topic(topicV1).create(); Producer producer2 = pulsarClient.newProducer().topic(topicV2).create(); From 15b75089625110a3182559b56a9b308c16ef80de Mon Sep 17 00:00:00 2001 From: ken <1647023764@qq.com> Date: Tue, 5 Sep 2023 16:21:23 +0800 Subject: [PATCH 271/494] [fix][broker] revert remove duplicate topics name when deleteNamespace (#21087) --- .../org/apache/pulsar/broker/admin/impl/NamespacesBase.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java index 625cb05c7d3bf..cb7fd1f32f2ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/NamespacesBase.java @@ -280,12 +280,6 @@ private void internalRetryableDeleteNamespaceAsync0(boolean force, int retryTime return old; }); } - allUserCreatedTopics.removeIf(t -> - allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); - allSystemTopics.removeIf(t -> - allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); - topicPolicy.removeIf(t -> - allPartitionedTopics.contains(TopicName.get(t).getPartitionedTopicName())); return markDeleteFuture.thenCompose(__ -> internalDeleteTopicsAsync(allUserCreatedTopics)) .thenCompose(ignore -> From 25cc204d26bc23deaeaba9d834f8c6f7e4d3fbff Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Mon, 11 Sep 2023 09:55:21 +0800 Subject: [PATCH 272/494] [improve][broker] Upgrade bookkeeper to 4.16.3 (#21146) --- .../server/src/assemble/LICENSE.bin.txt | 56 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 6 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 30 +++++----- 4 files changed, 47 insertions(+), 47 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 3d84a59a735ce..bdf75ab46049e 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -346,34 +346,34 @@ The Apache Software License, Version 2.0 - net.java.dev.jna-jna-jpms-5.12.1.jar - net.java.dev.jna-jna-platform-jpms-5.12.1.jar * BookKeeper - - org.apache.bookkeeper-bookkeeper-common-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-proto-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-server-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.2.jar - - org.apache.bookkeeper-circe-checksum-4.16.2.jar - - org.apache.bookkeeper-cpu-affinity-4.16.2.jar - - org.apache.bookkeeper-statelib-4.16.2.jar - - org.apache.bookkeeper-stream-storage-api-4.16.2.jar - - org.apache.bookkeeper-stream-storage-common-4.16.2.jar - - org.apache.bookkeeper-stream-storage-java-client-4.16.2.jar - - org.apache.bookkeeper-stream-storage-java-client-base-4.16.2.jar - - org.apache.bookkeeper-stream-storage-proto-4.16.2.jar - - org.apache.bookkeeper-stream-storage-server-4.16.2.jar - - org.apache.bookkeeper-stream-storage-service-api-4.16.2.jar - - org.apache.bookkeeper-stream-storage-service-impl-4.16.2.jar - - org.apache.bookkeeper.http-http-server-4.16.2.jar - - org.apache.bookkeeper.http-vertx-http-server-4.16.2.jar - - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.2.jar - - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.2.jar - - org.apache.distributedlog-distributedlog-common-4.16.2.jar - - org.apache.distributedlog-distributedlog-core-4.16.2-tests.jar - - org.apache.distributedlog-distributedlog-core-4.16.2.jar - - org.apache.distributedlog-distributedlog-protocol-4.16.2.jar - - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.2.jar - - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.2.jar - - org.apache.bookkeeper-native-io-4.16.2.jar + - org.apache.bookkeeper-bookkeeper-common-4.16.3.jar + - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.3.jar + - org.apache.bookkeeper-bookkeeper-proto-4.16.3.jar + - org.apache.bookkeeper-bookkeeper-server-4.16.3.jar + - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.3.jar + - org.apache.bookkeeper-circe-checksum-4.16.3.jar + - org.apache.bookkeeper-cpu-affinity-4.16.3.jar + - org.apache.bookkeeper-statelib-4.16.3.jar + - org.apache.bookkeeper-stream-storage-api-4.16.3.jar + - org.apache.bookkeeper-stream-storage-common-4.16.3.jar + - org.apache.bookkeeper-stream-storage-java-client-4.16.3.jar + - org.apache.bookkeeper-stream-storage-java-client-base-4.16.3.jar + - org.apache.bookkeeper-stream-storage-proto-4.16.3.jar + - org.apache.bookkeeper-stream-storage-server-4.16.3.jar + - org.apache.bookkeeper-stream-storage-service-api-4.16.3.jar + - org.apache.bookkeeper-stream-storage-service-impl-4.16.3.jar + - org.apache.bookkeeper.http-http-server-4.16.3.jar + - org.apache.bookkeeper.http-vertx-http-server-4.16.3.jar + - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.3.jar + - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.3.jar + - org.apache.distributedlog-distributedlog-common-4.16.3.jar + - org.apache.distributedlog-distributedlog-core-4.16.3-tests.jar + - org.apache.distributedlog-distributedlog-core-4.16.3.jar + - org.apache.distributedlog-distributedlog-protocol-4.16.3.jar + - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.3.jar + - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.3.jar + - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.3.jar + - org.apache.bookkeeper-native-io-4.16.3.jar * Apache HTTP Client - org.apache.httpcomponents-httpclient-4.5.13.jar - org.apache.httpcomponents-httpcore-4.4.15.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 7e275b6502531..17995c1cd0664 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -390,9 +390,9 @@ The Apache Software License, Version 2.0 - log4j-web-2.18.0.jar * BookKeeper - - bookkeeper-common-allocator-4.16.2.jar - - cpu-affinity-4.16.2.jar - - circe-checksum-4.16.2.jar + - bookkeeper-common-allocator-4.16.3.jar + - cpu-affinity-4.16.3.jar + - circe-checksum-4.16.3.jar * AirCompressor - aircompressor-0.20.jar * AsyncHttpClient diff --git a/pom.xml b/pom.xml index d5d2fdcd00f69..a6a4ca2abb532 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ flexible messaging model and an intuitive client API. 1.21 - 4.16.2 + 4.16.3 3.8.1 1.5.0 1.10.0 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index cfbd2e7d59253..2a2a6f5e91fcf 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -426,21 +426,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.16.2.jar - - bookkeeper-common-allocator-4.16.2.jar - - bookkeeper-proto-4.16.2.jar - - bookkeeper-server-4.16.2.jar - - bookkeeper-stats-api-4.16.2.jar - - bookkeeper-tools-framework-4.16.2.jar - - circe-checksum-4.16.2.jar - - codahale-metrics-provider-4.16.2.jar - - cpu-affinity-4.16.2.jar - - http-server-4.16.2.jar - - prometheus-metrics-provider-4.16.2.jar - - codahale-metrics-provider-4.16.2.jar - - bookkeeper-slogger-api-4.16.2.jar - - bookkeeper-slogger-slf4j-4.16.2.jar - - native-io-4.16.2.jar + - bookkeeper-common-4.16.3.jar + - bookkeeper-common-allocator-4.16.3.jar + - bookkeeper-proto-4.16.3.jar + - bookkeeper-server-4.16.3.jar + - bookkeeper-stats-api-4.16.3.jar + - bookkeeper-tools-framework-4.16.3.jar + - circe-checksum-4.16.3.jar + - codahale-metrics-provider-4.16.3.jar + - cpu-affinity-4.16.3.jar + - http-server-4.16.3.jar + - prometheus-metrics-provider-4.16.3.jar + - codahale-metrics-provider-4.16.3.jar + - bookkeeper-slogger-api-4.16.3.jar + - bookkeeper-slogger-slf4j-4.16.3.jar + - native-io-4.16.3.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar From 78b26eb4231f086c25065c24e6e88e52151a09ba Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 7 Sep 2023 03:15:22 +0800 Subject: [PATCH 273/494] [improve] [broker] Improve logs for troubleshooting (#21141) --- .../mledger/impl/ManagedCursorImpl.java | 26 +++++++++++-------- .../mledger/impl/NonDurableCursorImpl.java | 8 ++++-- .../mledger/impl/NonDurableCursorTest.java | 3 ++- .../service/persistent/PersistentTopic.java | 8 +++--- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 965e078bf02a8..2e0668639398a 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -509,7 +509,7 @@ public void operationComplete(ManagedCursorInfo info, Stat stat) { callback.operationComplete(); } else { // Need to proceed and read the last entry in the specified ledger to find out the last position - log.info("[{}] Consumer {} meta-data recover from ledger {}", ledger.getName(), name, + log.info("[{}] Cursor {} meta-data recover from ledger {}", ledger.getName(), name, info.getCursorsLedgerId()); recoverFromLedger(info, callback); } @@ -529,16 +529,16 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac long ledgerId = info.getCursorsLedgerId(); OpenCallback openCallback = (rc, lh, ctx) -> { if (log.isInfoEnabled()) { - log.info("[{}] Opened ledger {} for consumer {}. rc={}", ledger.getName(), ledgerId, name, rc); + log.info("[{}] Opened ledger {} for cursor {}. rc={}", ledger.getName(), ledgerId, name, rc); } if (isBkErrorNotRecoverable(rc)) { - log.error("[{}] Error opening metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, + log.error("[{}] Error opening metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc)); // Rewind to oldest entry available initialize(getRollbackPosition(info), Collections.emptyMap(), Collections.emptyMap(), callback); return; } else if (rc != BKException.Code.OK) { - log.warn("[{}] Error opening metadata ledger {} for consumer {}: {}", ledger.getName(), ledgerId, name, + log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc)); callback.operationFailed(new ManagedLedgerException(BKException.getMessage(rc))); return; @@ -548,7 +548,7 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac long lastEntryInLedger = lh.getLastAddConfirmed(); if (lastEntryInLedger < 0) { - log.warn("[{}] Error reading from metadata ledger {} for consumer {}: No entries in ledger", + log.warn("[{}] Error reading from metadata ledger {} for cursor {}: No entries in ledger", ledger.getName(), ledgerId, name); // Rewind to last cursor snapshot available initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback); @@ -560,13 +560,13 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac log.debug("[{}} readComplete rc={} entryId={}", ledger.getName(), rc1, lh1.getLastAddConfirmed()); } if (isBkErrorNotRecoverable(rc1)) { - log.error("[{}] Error reading from metadata ledger {} for consumer {}: {}", ledger.getName(), + log.error("[{}] Error reading from metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc1)); // Rewind to oldest entry available initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback); return; } else if (rc1 != BKException.Code.OK) { - log.warn("[{}] Error reading from metadata ledger {} for consumer {}: {}", ledger.getName(), + log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name, BKException.getMessage(rc1)); callback.operationFailed(createManagedLedgerException(rc1)); @@ -2454,8 +2454,12 @@ List filterReadEntries(List entries) { @Override public synchronized String toString() { - return MoreObjects.toStringHelper(this).add("ledger", ledger.getName()).add("name", name) - .add("ackPos", markDeletePosition).add("readPos", readPosition).toString(); + return MoreObjects.toStringHelper(this) + .add("ledger", ledger.getName()) + .add("name", name) + .add("ackPos", markDeletePosition) + .add("readPos", readPosition) + .toString(); } @Override @@ -3069,7 +3073,7 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin if (shouldCloseLedger(lh1)) { if (log.isDebugEnabled()) { - log.debug("[{}] Need to create new metadata ledger for consumer {}", ledger.getName(), name); + log.debug("[{}] Need to create new metadata ledger for cursor {}", ledger.getName(), name); } startCreatingNewMetadataLedger(); } @@ -3154,7 +3158,7 @@ public void operationComplete(Void result, Stat stat) { @Override public void operationFailed(MetaStoreException e) { - log.warn("[{}] Failed to update consumer {}", ledger.getName(), name, e); + log.warn("[{}] Failed to update cursor metadata {}", ledger.getName(), name, e); // it means it failed to switch the newly created ledger so, it should be // deleted to prevent leak deleteLedgerAsync(lh).thenRun(() -> callback.operationFailed(e)); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java index 9d2829b1707f4..51e56158cad55 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java @@ -138,8 +138,12 @@ public void rewind() { @Override public synchronized String toString() { - return MoreObjects.toStringHelper(this).add("ledger", ledger.getName()).add("ackPos", markDeletePosition) - .add("readPos", readPosition).toString(); + return MoreObjects.toStringHelper(this) + .add("ledger", ledger.getName()) + .add("cursor", getName()) + .add("ackPos", markDeletePosition) + .add("readPos", readPosition) + .toString(); } private static final Logger log = LoggerFactory.getLogger(NonDurableCursorImpl.class); diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java index 1ad3f5f8de631..1e1f7df0a46d5 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java @@ -80,7 +80,8 @@ void readFromEmptyLedger() throws Exception { entries.forEach(Entry::release); // Test string representation - assertEquals(c1.toString(), "NonDurableCursorImpl{ledger=my_test_ledger, ackPos=3:-1, readPos=3:1}"); + assertEquals(c1.toString(), "NonDurableCursorImpl{ledger=my_test_ledger, cursor=" + + c1.getName() + ", ackPos=3:-1, readPos=3:1}"); } @Test(timeOut = 20000) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 948c671044720..85c1d98e04736 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -901,8 +901,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St consumer.close(); } catch (BrokerServiceException e) { if (e instanceof ConsumerBusyException) { - log.warn("[{}][{}] Consumer {} {} already connected", - topic, subscriptionName, consumerId, consumerName); + log.warn("[{}][{}] Consumer {} {} already connected: {}", + topic, subscriptionName, consumerId, consumerName, e.getMessage()); } else if (e instanceof SubscriptionBusyException) { log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage()); } @@ -932,8 +932,8 @@ private CompletableFuture internalSubscribe(final TransportCnx cnx, St decrementUsageCount(); if (ex.getCause() instanceof ConsumerBusyException) { - log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, - consumerName); + log.warn("[{}][{}] Consumer {} {} already connected: {}", topic, subscriptionName, consumerId, + consumerName, ex.getCause().getMessage()); Consumer consumer = null; try { consumer = subscriptionFuture.isDone() ? getActiveConsumer(subscriptionFuture.get()) : null; From fe2c312e11fec1f36493f288502289adf12d3896 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 6 Sep 2023 20:27:23 -0500 Subject: [PATCH 274/494] [fix][broker] Fix web tls url null cause NPE (#21137) --- .../apache/pulsar/broker/rest/TopicsBase.java | 13 +- .../broker/admin/TopicsWithoutTlsTest.java | 128 ++++++++++++++++++ 2 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsWithoutTlsTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java index 33676a2c1657f..075c6227b7357 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/rest/TopicsBase.java @@ -53,6 +53,7 @@ import org.apache.avro.io.DecoderFactory; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.admin.impl.PersistentTopicsBase; import org.apache.pulsar.broker.authentication.AuthenticationParameters; @@ -432,8 +433,10 @@ private CompletableFuture lookUpBrokerForTopic(TopicName partitionedTopicN } LookupResult result = optionalResult.get(); - if (result.getLookupData().getHttpUrl().equals(pulsar().getWebServiceAddress()) - || result.getLookupData().getHttpUrlTls().equals(pulsar().getWebServiceAddressTls())) { + String httpUrl = result.getLookupData().getHttpUrl(); + String httpUrlTls = result.getLookupData().getHttpUrlTls(); + if ((StringUtils.isNotBlank(httpUrl) && httpUrl.equals(pulsar().getWebServiceAddress())) + || (StringUtils.isNotBlank(httpUrlTls) && httpUrlTls.equals(pulsar().getWebServiceAddressTls()))) { // Current broker owns the topic, add to owning topic. if (log.isDebugEnabled()) { log.debug("Complete topic look up for rest produce message request for topic {}, " @@ -454,12 +457,10 @@ private CompletableFuture lookUpBrokerForTopic(TopicName partitionedTopicN } if (result.isRedirect()) { // Redirect lookup. - completeLookup(Pair.of(Arrays.asList(result.getLookupData().getHttpUrl(), - result.getLookupData().getHttpUrlTls()), false), redirectAddresses, future); + completeLookup(Pair.of(Arrays.asList(httpUrl, httpUrlTls), false), redirectAddresses, future); } else { // Found owner for topic. - completeLookup(Pair.of(Arrays.asList(result.getLookupData().getHttpUrl(), - result.getLookupData().getHttpUrlTls()), true), redirectAddresses, future); + completeLookup(Pair.of(Arrays.asList(httpUrl, httpUrlTls), true), redirectAddresses, future); } } }).exceptionally(exception -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsWithoutTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsWithoutTlsTest.java new file mode 100644 index 0000000000000..88bf2f8f42108 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsWithoutTlsTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import lombok.Cleanup; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationDataHttps; +import org.apache.pulsar.broker.rest.Topics; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.TopicDomain; +import org.apache.pulsar.common.policies.data.ClusterDataImpl; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.websocket.data.ProducerMessage; +import org.apache.pulsar.websocket.data.ProducerMessages; +import org.mockito.ArgumentCaptor; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import javax.ws.rs.container.AsyncResponse; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; + +public class TopicsWithoutTlsTest extends MockedPulsarServiceBaseTest { + + private Topics topics; + private final String testLocalCluster = "test"; + private final String testTenant = "my-tenant"; + private final String testNamespace = "my-namespace"; + private final String testTopicName = "my-topic"; + + @Override + @BeforeMethod + protected void setup() throws Exception { + super.internalSetup(); + topics = spy(new Topics()); + topics.setPulsar(pulsar); + doReturn(TopicDomain.persistent.value()).when(topics).domain(); + doReturn("test-app").when(topics).clientAppId(); + doReturn(mock(AuthenticationDataHttps.class)).when(topics).clientAuthData(); + admin.clusters().createCluster(testLocalCluster, new ClusterDataImpl()); + admin.tenants().createTenant(testTenant, new TenantInfoImpl(Set.of("role1", "role2"), Set.of(testLocalCluster))); + admin.namespaces().createNamespace(testTenant + "/" + testNamespace, + Set.of(testLocalCluster)); + } + + @Override + protected void doInitConf() throws Exception { + super.doInitConf(); + this.conf.setBrokerServicePortTls(Optional.empty()); + this.conf.setWebServicePortTls(Optional.empty()); + } + + @Override + @AfterMethod + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testLookUpWithRedirect() throws Exception { + String topicName = "persistent://" + testTenant + "/" + testNamespace + "/" + testTopicName; + URI requestPath = URI.create(pulsar.getWebServiceAddress() + "/topics/my-tenant/my-namespace/my-topic"); + //create topic on one broker + admin.topics().createNonPartitionedTopic(topicName); + conf.setBrokerServicePort(Optional.of(0)); + conf.setWebServicePort(Optional.of(0)); + @Cleanup + PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf); + PulsarService pulsar2 = pulsarTestContext2.getPulsarService(); + doReturn(false).when(topics).isRequestHttps(); + UriInfo uriInfo = mock(UriInfo.class); + doReturn(requestPath).when(uriInfo).getRequestUri(); + FieldUtils.writeField(topics, "uri", uriInfo, true); + //do produce on another broker + topics.setPulsar(pulsar2); + AsyncResponse asyncResponse = mock(AsyncResponse.class); + ProducerMessages producerMessages = new ProducerMessages(); + producerMessages.setValueSchema(ObjectMapperFactory.getMapper().getObjectMapper(). + writeValueAsString(Schema.INT64.getSchemaInfo())); + String message = "[]"; + producerMessages.setMessages(createMessages(message)); + topics.produceOnPersistentTopic(asyncResponse, testTenant, testNamespace, testTopicName, false, producerMessages); + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(Response.class); + verify(asyncResponse, timeout(5000).times(1)).resume(responseCaptor.capture()); + // Verify got redirect response + Assert.assertEquals(responseCaptor.getValue().getStatusInfo(), Response.Status.TEMPORARY_REDIRECT); + // Verify URI point to address of broker the topic was created on + Assert.assertEquals(responseCaptor.getValue().getLocation().toString(), requestPath.toString()); + } + + private static List createMessages(String message) throws JsonProcessingException { + return ObjectMapperFactory.getMapper().reader() + .forType(new TypeReference>() { + }).readValue(message); + } +} From b104369207844c06d114dd979ccdea32a6425929 Mon Sep 17 00:00:00 2001 From: erobot Date: Mon, 11 Sep 2023 09:56:34 +0800 Subject: [PATCH 275/494] [fix][broker] Fix unack count when mixing non batch index and batch index acks (#21126) --- .../pulsar/broker/service/Consumer.java | 9 ++--- .../BatchMessageWithBatchIndexLevelTest.java | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 71a5bc429d117..cedb2c0f23f97 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -491,7 +491,6 @@ public CompletableFuture messageAcked(CommandAck ack) { private CompletableFuture individualAckNormal(CommandAck ack, Map properties) { List positionsAcked = new ArrayList<>(); long totalAckCount = 0; - boolean individualAck = false; for (int i = 0; i < ack.getMessageIdsCount(); i++) { MessageIdData msgId = ack.getMessageIdAt(i); PositionImpl position; @@ -512,19 +511,15 @@ private CompletableFuture individualAckNormal(CommandAck ack, Map producer = pulsarClient.newProducer() + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(100, TimeUnit.MILLISECONDS) + .create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topicName) + .subscriptionName("sub") + .subscriptionType(SubscriptionType.Shared) + .acknowledgmentGroupTime(100, TimeUnit.MILLISECONDS) + .enableBatchIndexAcknowledgment(true) + .isAckReceiptEnabled(true) + .subscribe(); + + // send two batch messages: [(1), (2,3)] + producer.send("1".getBytes()); + producer.sendAsync("2".getBytes()); + producer.send("3".getBytes()); + + Message message1 = consumer.receive(); + Message message2 = consumer.receive(); + Message message3 = consumer.receive(); + consumer.acknowledgeAsync(message1); + consumer.acknowledge(message2); // send group ack: non-index ack for 1, index ack for 2 + consumer.acknowledge(message3); // index ack for 3 + + assertEquals(admin.topics().getStats(topicName).getSubscriptions() + .get("sub").getUnackedMessages(), 0); + } } From 635b5e569162c847735016d51d562c1c3750c01f Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Tue, 5 Sep 2023 08:12:49 +0800 Subject: [PATCH 276/494] [fix] [broker] Fix isolated group not work problem. (#21096) When upgraded the pulsar version from 2.9.2 to 2.10.3, and the isolated group feature not work anymore. Finally, we found the problem. In IsolatedBookieEnsemblePlacementPolicy, when it gets the bookie rack from the metadata store cache, uses future.isDone() to avoid sync operation. If the future is incomplete, return empty blacklists. The cache may expire due to the caffeine cache `getExpireAfterWriteMillis` config, if the cache expires, the future may be incomplete. (#21095 will correct the behavior) In 2.9.2, it uses the sync to get data from the metadata store, we should also keep the behavior. --- ...IsolatedBookieEnsemblePlacementPolicy.java | 31 +++-- ...atedBookieEnsemblePlacementPolicyTest.java | 115 ++++++++++++++++++ 2 files changed, 134 insertions(+), 12 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java index 2594798485a20..c0645f87936e5 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicy.java @@ -19,6 +19,7 @@ package org.apache.pulsar.bookie.rackawareness; import static org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping.METADATA_STORE_INSTANCE; +import com.google.common.annotations.VisibleForTesting; import io.netty.util.HashedWheelTimer; import java.util.Arrays; import java.util.Collections; @@ -27,7 +28,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.CompletableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; @@ -61,6 +61,7 @@ public class IsolatedBookieEnsemblePlacementPolicy extends RackawareEnsemblePlac private static final String PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP = "*"; + private volatile BookiesRackConfiguration cachedRackConfiguration = null; public IsolatedBookieEnsemblePlacementPolicy() { super(); @@ -86,7 +87,12 @@ public RackawareEnsemblePlacementPolicyImpl initialize(ClientConfiguration conf, } // Only add the bookieMappingCache if we have defined an isolation group bookieMappingCache = store.getMetadataCache(BookiesRackConfiguration.class); - bookieMappingCache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH).join(); + bookieMappingCache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH).thenAccept(opt -> opt.ifPresent( + bookiesRackConfiguration -> cachedRackConfiguration = bookiesRackConfiguration)) + .exceptionally(e -> { + log.warn("Failed to load bookies rack configuration while initialize the PlacementPolicy."); + return null; + }); } if (conf.getProperty(SECONDARY_ISOLATION_BOOKIE_GROUPS) != null) { String secondaryIsolationGroupsString = ConfigurationStringUtil @@ -179,25 +185,26 @@ private static Pair, Set> getIsolationGroup( return pair; } - private Set getExcludedBookiesWithIsolationGroups(int ensembleSize, + @VisibleForTesting + Set getExcludedBookiesWithIsolationGroups(int ensembleSize, Pair, Set> isolationGroups) { Set excludedBookies = new HashSet<>(); - if (isolationGroups != null && isolationGroups.getLeft().contains(PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP)) { + if (isolationGroups != null && isolationGroups.getLeft().contains(PULSAR_SYSTEM_TOPIC_ISOLATION_GROUP)) { return excludedBookies; } try { if (bookieMappingCache != null) { - CompletableFuture> future = - bookieMappingCache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH); + bookieMappingCache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH) + .thenAccept(opt -> cachedRackConfiguration = opt.orElse(null)).exceptionally(e -> { + log.warn("Failed to update the newest bookies rack config."); + return null; + }); - Optional optRes = (future.isDone() && !future.isCompletedExceptionally()) - ? future.join() : Optional.empty(); - - if (optRes.isEmpty()) { + BookiesRackConfiguration allGroupsBookieMapping = cachedRackConfiguration; + if (allGroupsBookieMapping == null) { + log.debug("The bookies rack config is not available at now."); return excludedBookies; } - - BookiesRackConfiguration allGroupsBookieMapping = optRes.get(); Set allBookies = allGroupsBookieMapping.keySet(); int totalAvailableBookiesInPrimaryGroup = 0; Set primaryIsolationGroup = Collections.emptySet(); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java index f535ced08f731..beb00197e4e9a 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/IsolatedBookieEnsemblePlacementPolicyTest.java @@ -18,11 +18,14 @@ */ package org.apache.pulsar.bookie.rackawareness; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import io.netty.util.HashedWheelTimer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -34,18 +37,23 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.feature.SettableFeatureProvider; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.common.policies.data.BookieInfo; +import org.apache.pulsar.common.policies.data.BookiesRackConfiguration; import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreFactory; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.cache.impl.MetadataCacheImpl; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -114,6 +122,113 @@ public void testNonRegionBookie() throws Exception { assertFalse(ensemble.contains(new BookieSocketAddress(BOOKIE4).toBookieId())); } + @Test + public void testMetadataStoreCases() throws Exception { + Map mainBookieGroup = new HashMap<>(); + mainBookieGroup.put(BOOKIE1, BookieInfo.builder().rack("rack0").build()); + mainBookieGroup.put(BOOKIE2, BookieInfo.builder().rack("rack1").build()); + mainBookieGroup.put(BOOKIE3, BookieInfo.builder().rack("rack1").build()); + mainBookieGroup.put(BOOKIE4, BookieInfo.builder().rack("rack0").build()); + + Map secondaryBookieGroup = new HashMap<>(); + + store = mock(MetadataStoreExtended.class); + MetadataCacheImpl cache = mock(MetadataCacheImpl.class); + when(store.getMetadataCache(BookiesRackConfiguration.class)).thenReturn(cache); + CompletableFuture> initialFuture = new CompletableFuture<>(); + //The initialFuture only has group1. + BookiesRackConfiguration rackConfiguration1 = new BookiesRackConfiguration(); + rackConfiguration1.put("group1", mainBookieGroup); + rackConfiguration1.put("group2", secondaryBookieGroup); + initialFuture.complete(Optional.of(rackConfiguration1)); + + long waitTime = 2000; + CompletableFuture> waitingCompleteFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(waitTime); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + //The waitingCompleteFuture has group1 and group2. + BookiesRackConfiguration rackConfiguration2 = new BookiesRackConfiguration(); + Map mainBookieGroup2 = new HashMap<>(); + mainBookieGroup2.put(BOOKIE1, BookieInfo.builder().rack("rack0").build()); + mainBookieGroup2.put(BOOKIE2, BookieInfo.builder().rack("rack1").build()); + mainBookieGroup2.put(BOOKIE4, BookieInfo.builder().rack("rack0").build()); + + Map secondaryBookieGroup2 = new HashMap<>(); + secondaryBookieGroup2.put(BOOKIE3, BookieInfo.builder().rack("rack0").build()); + rackConfiguration2.put("group1", mainBookieGroup2); + rackConfiguration2.put("group2", secondaryBookieGroup2); + waitingCompleteFuture.complete(Optional.of(rackConfiguration2)); + }).start(); + + long longWaitTime = 4000; + CompletableFuture> emptyFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(longWaitTime); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + //The emptyFuture means that the zk node /bookies already be removed. + emptyFuture.complete(Optional.empty()); + }).start(); + + //Return different future means that cache expire. + when(cache.get(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH)) + .thenReturn(initialFuture).thenReturn(initialFuture) + .thenReturn(waitingCompleteFuture).thenReturn(waitingCompleteFuture) + .thenReturn(emptyFuture).thenReturn(emptyFuture); + + IsolatedBookieEnsemblePlacementPolicy isolationPolicy = new IsolatedBookieEnsemblePlacementPolicy(); + ClientConfiguration bkClientConf = new ClientConfiguration(); + bkClientConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); + bkClientConf.setProperty(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, isolationGroups); + isolationPolicy.initialize(bkClientConf, Optional.empty(), timer, SettableFeatureProvider.DISABLE_ALL, NullStatsLogger.INSTANCE, BookieSocketAddress.LEGACY_BOOKIEID_RESOLVER); + isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies); + + MutablePair, Set> groups = new MutablePair<>(); + groups.setLeft(Sets.newHashSet("group1")); + groups.setRight(new HashSet<>()); + + //initialFuture, the future is waiting done. + Set blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertTrue(blacklist.isEmpty()); + + //waitingCompleteFuture, the future is waiting done. + blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertTrue(blacklist.isEmpty()); + + Thread.sleep(waitTime); + + //waitingCompleteFuture, the future is already done. + blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertFalse(blacklist.isEmpty()); + assertEquals(blacklist.size(), 1); + BookieId excludeBookie = blacklist.iterator().next(); + assertEquals(excludeBookie.toString(), BOOKIE3); + + //emptyFuture, the future is waiting done. + blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertFalse(blacklist.isEmpty()); + assertEquals(blacklist.size(), 1); + excludeBookie = blacklist.iterator().next(); + assertEquals(excludeBookie.toString(), BOOKIE3); + + Thread.sleep(longWaitTime - waitTime); + + //emptyFuture, the future is already done. + blacklist = + isolationPolicy.getExcludedBookiesWithIsolationGroups(2, groups); + assertTrue(blacklist.isEmpty()); + } + @Test public void testBasic() throws Exception { Map> bookieMapping = new HashMap<>(); From 242e29da852487ce81aa34a435d97aea4e4b79bc Mon Sep 17 00:00:00 2001 From: Apurva007 Date: Tue, 5 Sep 2023 07:46:30 -0700 Subject: [PATCH 277/494] [fix][proxy] Fix Proxy 502 gateway error when it is configured with Keystore TLS and admin API is called (#21077) --- .../proxy/server/AdminProxyHandler.java | 41 +++-- .../AdminProxyHandlerKeystoreTLSTest.java | 142 ++++++++++++++++++ 2 files changed, 171 insertions(+), 12 deletions(-) create mode 100644 pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerKeystoreTLSTest.java diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index d9dda9823ea89..c528ceb2cf5b7 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -41,8 +41,10 @@ import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.api.KeyStoreParams; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.util.SecurityUtility; +import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.HttpRequest; @@ -269,20 +271,35 @@ protected HttpClient newHttpClient() { SSLContext sslCtx; AuthenticationDataProvider authData = auth.getAuthData(); - if (authData.hasDataForTls()) { - sslCtx = SecurityUtility.createSslContext( + if (config.isBrokerClientTlsEnabledWithKeyStore()) { + KeyStoreParams params = authData.hasDataForTls() ? authData.getTlsKeyStoreParams() : null; + sslCtx = KeyStoreSSLContext.createClientSslContext( + config.getBrokerClientSslProvider(), + params != null ? params.getKeyStoreType() : null, + params != null ? params.getKeyStorePath() : null, + params != null ? params.getKeyStorePassword() : null, config.isTlsAllowInsecureConnection(), - trustCertificates, - authData.getTlsCertificates(), - authData.getTlsPrivateKey(), - config.getBrokerClientSslProvider() - ); + config.getBrokerClientTlsTrustStoreType(), + config.getBrokerClientTlsTrustStore(), + config.getBrokerClientTlsTrustStorePassword(), + config.getBrokerClientTlsCiphers(), + config.getBrokerClientTlsProtocols()); } else { - sslCtx = SecurityUtility.createSslContext( - config.isTlsAllowInsecureConnection(), - trustCertificates, - config.getBrokerClientSslProvider() - ); + if (authData.hasDataForTls()) { + sslCtx = SecurityUtility.createSslContext( + config.isTlsAllowInsecureConnection(), + trustCertificates, + authData.getTlsCertificates(), + authData.getTlsPrivateKey(), + config.getBrokerClientSslProvider() + ); + } else { + sslCtx = SecurityUtility.createSslContext( + config.isTlsAllowInsecureConnection(), + trustCertificates, + config.getBrokerClientSslProvider() + ); + } } SslContextFactory contextFactory = new SslContextFactory.Client(); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerKeystoreTLSTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerKeystoreTLSTest.java new file mode 100644 index 0000000000000..d6796b7eaa6d2 --- /dev/null +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerKeystoreTLSTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.proxy.server; + +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; +import org.apache.pulsar.broker.authentication.AuthenticationService; +import org.apache.pulsar.broker.resources.PulsarResources; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.impl.auth.AuthenticationKeyStoreTls; +import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; +import org.apache.pulsar.policies.data.loadbalancer.LoadReport; +import org.eclipse.jetty.servlet.ServletHolder; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +public class AdminProxyHandlerKeystoreTLSTest extends MockedPulsarServiceBaseTest { + + + private final ProxyConfiguration proxyConfig = new ProxyConfiguration(); + + private WebServer webServer; + + private BrokerDiscoveryProvider discoveryProvider; + + private PulsarResources resource; + + @BeforeMethod + @Override + protected void setup() throws Exception { + + conf.setAuthenticationEnabled(true); + conf.setAuthorizationEnabled(false); + conf.setWebServicePortTls(Optional.of(0)); + conf.setBrokerServicePortTls(Optional.of(0)); + conf.setTlsEnabledWithKeyStore(true); + conf.setTlsAllowInsecureConnection(false); + conf.setTlsKeyStoreType(KEYSTORE_TYPE); + conf.setTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + conf.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); + conf.setTlsTrustStoreType(KEYSTORE_TYPE); + conf.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); + conf.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + + super.internalSetup(); + + proxyConfig.setWebServicePort(Optional.of(0)); + proxyConfig.setBrokerProxyAllowedTargetPorts("*"); + proxyConfig.setServicePortTls(Optional.of(0)); + proxyConfig.setWebServicePortTls(Optional.of(0)); + proxyConfig.setTlsEnabledWithBroker(true); + proxyConfig.setTlsEnabledWithKeyStore(true); + + proxyConfig.setTlsKeyStoreType(KEYSTORE_TYPE); + proxyConfig.setTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + proxyConfig.setTlsKeyStorePassword(BROKER_KEYSTORE_PW); + proxyConfig.setTlsTrustStoreType(KEYSTORE_TYPE); + proxyConfig.setTlsTrustStore(CLIENT_TRUSTSTORE_FILE_PATH); + proxyConfig.setTlsTrustStorePassword(CLIENT_TRUSTSTORE_PW); + + proxyConfig.setMetadataStoreUrl(DUMMY_VALUE); + proxyConfig.setConfigurationMetadataStoreUrl(GLOBAL_DUMMY_VALUE); + proxyConfig.setBrokerClientTlsEnabledWithKeyStore(true); + proxyConfig.setBrokerClientTlsKeyStoreType(KEYSTORE_TYPE); + proxyConfig.setBrokerClientTlsKeyStore(BROKER_KEYSTORE_FILE_PATH); + proxyConfig.setBrokerClientTlsKeyStorePassword(BROKER_KEYSTORE_PW); + proxyConfig.setBrokerClientTlsTrustStoreType(KEYSTORE_TYPE); + proxyConfig.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); + proxyConfig.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); + Set providers = new HashSet<>(); + providers.add(AuthenticationProviderTls.class.getName()); + proxyConfig.setAuthenticationProviders(providers); + proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationKeyStoreTls.class.getName()); + proxyConfig.setBrokerClientAuthenticationParameters(String.format("keyStoreType:%s,keyStorePath:%s,keyStorePassword:%s", + KEYSTORE_TYPE, BROKER_KEYSTORE_FILE_PATH, BROKER_KEYSTORE_PW)); + + resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), + new ZKMetadataStore(mockZooKeeperGlobal)); + webServer = new WebServer(proxyConfig, new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig))); + discoveryProvider = spy(new BrokerDiscoveryProvider(proxyConfig, resource)); + LoadManagerReport report = new LoadReport(brokerUrl.toString(), brokerUrlTls.toString(), null, null); + doReturn(report).when(discoveryProvider).nextBroker(); + ServletHolder servletHolder = new ServletHolder(new AdminProxyHandler(proxyConfig, discoveryProvider)); + webServer.addServlet("/admin", servletHolder); + webServer.addServlet("/lookup", servletHolder); + webServer.start(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + webServer.stop(); + super.internalCleanup(); + } + + PulsarAdmin getAdminClient() throws Exception { + return PulsarAdmin.builder() + .serviceHttpUrl("https://localhost:" + webServer.getListenPortHTTPS().get()) + .useKeyStoreTls(true) + .allowTlsInsecureConnection(false) + .tlsTrustStorePath(BROKER_TRUSTSTORE_FILE_PATH) + .tlsTrustStorePassword(BROKER_TRUSTSTORE_PW) + .authentication(AuthenticationKeyStoreTls.class.getName(), + String.format("keyStoreType:%s,keyStorePath:%s,keyStorePassword:%s", + KEYSTORE_TYPE, BROKER_KEYSTORE_FILE_PATH, BROKER_KEYSTORE_PW)) + .build(); + } + + @Test + public void testAdmin() throws Exception { + getAdminClient().clusters().createCluster(configClusterName, ClusterData.builder().serviceUrl(brokerUrl.toString()).build()); + } + +} From 086c59ac0b5254bb510da6faefeab07d1da4ca1d Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 5 Jul 2023 12:17:27 +0800 Subject: [PATCH 278/494] [fix][sec] Upgrade Guava to 32.1.1 to address CVE-2023-2976 (#20699) --- buildtools/pom.xml | 2 +- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index eab01fd0c51f1..b8595959853f0 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -49,7 +49,7 @@ 3.1.2 4.1.94.Final 4.2.3 - 32.0.0-jre + 32.1.1-jre 1.10.12 2.0 3.12.4 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index bdf75ab46049e..9476793d869c1 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -265,7 +265,7 @@ The Apache Software License, Version 2.0 - com.google.code.gson-gson-2.8.9.jar - io.gsonfire-gson-fire-1.8.5.jar * Guava - - com.google.guava-guava-32.0.0-jre.jar + - com.google.guava-guava-32.1.1-jre.jar - com.google.guava-failureaccess-1.0.1.jar - com.google.guava-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- com.google.j2objc-j2objc-annotations-1.3.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 17995c1cd0664..66ee8dbb4519e 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -326,7 +326,7 @@ The Apache Software License, Version 2.0 * Gson - gson-2.8.9.jar * Guava - - guava-32.0.0-jre.jar + - guava-32.1.1-jre.jar - failureaccess-1.0.1.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar * J2ObjC Annotations -- j2objc-annotations-1.3.jar diff --git a/pom.xml b/pom.xml index a6a4ca2abb532..5bf2d2df8453d 100644 --- a/pom.xml +++ b/pom.xml @@ -202,7 +202,7 @@ flexible messaging model and an intuitive client API. 2.10.2 3.3.4 2.4.15 - 32.0.0-jre + 32.1.1-jre 1.0 0.16.1 6.2.8 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 2a2a6f5e91fcf..ea4b3fe93b89f 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -221,7 +221,7 @@ The Apache Software License, Version 2.0 - jackson-module-jaxb-annotations-2.14.2.jar - jackson-module-jsonSchema-2.14.2.jar * Guava - - guava-32.0.0-jre.jar + - guava-32.1.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar - failureaccess-1.0.1.jar * Google Guice diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 3d4f46977f250..120d0d3579eb0 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -37,7 +37,7 @@ 2.6 0.0.12 3.0.5 - 32.0.0-jre + 32.1.1-jre 2.12.1 2.5.1 4.0.1 From b510415538b7ba05f7a02b0a0214749864c72772 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 11 Sep 2023 21:06:05 +0800 Subject: [PATCH 279/494] [fix] [client] fix same producer/consumer use more than one connection per broker (#21144) Motivation: Pulsar has two mechanisms to guarantee that a producer connects to the broker multiple times the result is still correct. - In a connection, the second connection waits for the first connection to complete. - In a topic, the second connection will override the previous one. However, if a producer can use different connections to connect to the broker, these two mechanisms will not work. When the config `connectionsPerBroker` of `PulsarClient` is larger than `1`, a producer could use more than one connection, leading to the error above. You can reproduce this issue by the test `testSelectConnectionForSameProducer.` Modifications: Make the same producer/consumer usage the same connection (cherry picked from commit f2b9a3fffd4aad414ba5f35ebff17502cb64eb12) --- .../impl/TransactionBufferHandlerImpl.java | 8 +++- .../service/AbstractReplicatorTest.java | 4 ++ .../broker/service/PersistentTopicTest.java | 3 ++ .../NonStartableTestPulsarService.java | 7 ++- .../buffer/TransactionBufferClientTest.java | 22 ++++++++-- .../TransactionBufferHandlerImplTest.java | 16 +++++-- .../client/impl/ConnectionPoolTest.java | 43 +++++++++++++++++-- .../pulsar/client/impl/PulsarTestClient.java | 2 +- .../pulsar/compaction/CompactorTest.java | 8 +++- .../pulsar/client/impl/ConnectionHandler.java | 6 ++- .../pulsar/client/impl/ConnectionPool.java | 16 +++++-- .../pulsar/client/impl/PulsarClientImpl.java | 19 ++++++-- .../AcknowledgementsGroupingTrackerTest.java | 2 + .../client/impl/AutoClusterFailoverTest.java | 9 ++++ .../impl/BatchMessageContainerImplTest.java | 4 ++ .../client/impl/ClientTestFixtures.java | 11 ++++- .../client/impl/ConsumerBuilderImplTest.java | 4 ++ .../impl/ControlledClusterFailoverTest.java | 3 ++ .../impl/PartitionedProducerImplTest.java | 2 + .../client/impl/ProducerBuilderImplTest.java | 2 + .../impl/ProducerStatsRecorderImplTest.java | 4 ++ .../client/impl/PulsarClientImplTest.java | 3 +- .../client/impl/TableViewBuilderImplTest.java | 2 + .../pulsar/client/impl/TableViewImplTest.java | 2 + .../client/impl/TopicListWatcherTest.java | 10 ++++- .../impl/UnAckedMessageTrackerTest.java | 4 ++ .../MultiVersionSchemaInfoProviderTest.java | 3 ++ .../functions/instance/ContextImplTest.java | 3 ++ .../pulsar/functions/sink/PulsarSinkTest.java | 4 ++ .../functions/source/PulsarSourceTest.java | 5 +++ .../functions/worker/LeaderServiceTest.java | 4 +- .../worker/MembershipManagerTest.java | 4 +- 32 files changed, 207 insertions(+), 32 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java index 48dcf259edb1b..625d27329d329 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TransactionBufferHandlerImpl.java @@ -61,6 +61,8 @@ public class TransactionBufferHandlerImpl implements TransactionBufferHandler { private final PulsarService pulsarService; private final PulsarClientImpl pulsarClient; + private final int randomKeyForSelectConnection; + private static final AtomicIntegerFieldUpdater REQUEST_CREDITS_UPDATER = AtomicIntegerFieldUpdater.newUpdater(TransactionBufferHandlerImpl.class, "requestCredits"); private volatile int requestCredits; @@ -74,6 +76,7 @@ public TransactionBufferHandlerImpl(PulsarService pulsarService, HashedWheelTime this.operationTimeoutInMills = operationTimeoutInMills; this.timer = timer; this.requestCredits = Math.max(100, maxConcurrentRequests); + this.randomKeyForSelectConnection = pulsarClient.getCnxPool().genRandomKeyToSelectCon(); } @Override @@ -296,7 +299,7 @@ protected OpRequestSend newObject(Handle handle) { } public CompletableFuture getClientCnxWithLookup(String topic) { - return pulsarClient.getConnection(topic); + return pulsarClient.getConnection(topic, randomKeyForSelectConnection); } public CompletableFuture getClientCnx(String topic) { @@ -317,7 +320,8 @@ public CompletableFuture getClientCnx(String topic) { } InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved(uri.getHost(), uri.getPort()); - return pulsarClient.getConnection(brokerAddress, brokerAddress); + return pulsarClient.getConnection(brokerAddress, brokerAddress, + randomKeyForSelectConnection); } else { // Bundle is unloading, lookup topic return getClientCnxWithLookup(topic); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java index 294a9b341ec69..f8034c37971cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/AbstractReplicatorTest.java @@ -40,6 +40,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; @@ -62,8 +63,11 @@ public void testRetryStartProducerStoppedByTopicRemove() throws Exception { final PulsarService pulsar = mock(PulsarService.class); final BrokerService broker = mock(BrokerService.class); final Topic localTopic = mock(Topic.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); final PulsarClientImpl localClient = mock(PulsarClientImpl.class); + when(localClient.getCnxPool()).thenReturn(connectionPool); final PulsarClientImpl remoteClient = mock(PulsarClientImpl.class); + when(remoteClient.getCnxPool()).thenReturn(connectionPool); final ProducerBuilder producerBuilder = mock(ProducerBuilder.class); final ConcurrentOpenHashMap>> topics = new ConcurrentOpenHashMap<>(); when(broker.executor()).thenReturn(eventLoopGroup); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index 02b10dd09a271..22b9949b7ee6c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -105,6 +105,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.transaction.TxnID; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.ProducerBuilderImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ProducerConfigurationData; @@ -1706,6 +1707,8 @@ public void testAtomicReplicationRemoval() throws Exception { ManagedCursor cursor = mock(ManagedCursorImpl.class); doReturn(remoteCluster).when(cursor).getName(); PulsarClientImpl pulsarClientMock = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClientMock.getCnxPool()).thenReturn(connectionPool); when(pulsarClientMock.newProducer(any())).thenAnswer( invocation -> { ProducerBuilderImpl producerBuilder = diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java index af365ed31934f..945a4c7ea2534 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/testcontext/NonStartableTestPulsarService.java @@ -20,6 +20,7 @@ import static org.apache.pulsar.broker.BrokerTestUtil.spyWithClassAndConstructorArgs; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import io.netty.channel.EventLoopGroup; import java.io.IOException; import java.util.Collections; @@ -42,6 +43,7 @@ import org.apache.pulsar.broker.transaction.buffer.TransactionBufferProvider; import org.apache.pulsar.broker.transaction.pendingack.TransactionPendingAckStoreProvider; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.common.naming.TopicName; @@ -76,7 +78,10 @@ public NonStartableTestPulsarService(SpyConfig spyConfig, ServiceConfiguration c throw new RuntimeException(e); } setSchemaRegistryService(spyWithClassAndConstructorArgs(DefaultSchemaRegistryService.class)); - setClient(mock(PulsarClientImpl.class)); + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); + setClient(mockClient); this.namespaceService = mock(NamespaceService.class); try { startNamespaceService(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index 2cfc9f46f0ebe..3873d9d37b20b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.transaction.buffer; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; @@ -51,6 +53,7 @@ import org.apache.pulsar.client.api.transaction.TransactionBufferClientException; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.naming.NamespaceName; @@ -253,14 +256,21 @@ public void testTransactionBufferMetrics() throws Exception { assertEquals(pending.size(), 1); } + /** + * This is a flaky test. + */ @Test public void testTransactionBufferClientTimeout() throws Exception { PulsarService pulsarService = pulsarServiceList.get(0); - PulsarClient mockClient = mock(PulsarClientImpl.class); + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); CompletableFuture completableFuture = new CompletableFuture<>(); ClientCnx clientCnx = mock(ClientCnx.class); completableFuture.complete(clientCnx); when(((PulsarClientImpl)mockClient).getConnection(anyString())).thenReturn(completableFuture); + when(((PulsarClientImpl)mockClient).getConnection(anyString(), anyInt())).thenReturn(completableFuture); + when(((PulsarClientImpl)mockClient).getConnection(any(), any(), anyInt())).thenReturn(completableFuture); ChannelHandlerContext cnx = mock(ChannelHandlerContext.class); when(clientCnx.ctx()).thenReturn(cnx); Channel channel = mock(Channel.class); @@ -287,7 +297,9 @@ public PulsarClient answer(InvocationOnMock invocation) throws Throwable { ConcurrentSkipListMap outstandingRequests = (ConcurrentSkipListMap) field.get(transactionBufferHandler); - assertEquals(outstandingRequests.size(), 1); + Awaitility.await().atMost(1, TimeUnit.SECONDS).untilAsserted(() -> { + assertEquals(outstandingRequests.size(), 1); + }); Awaitility.await().atLeast(2, TimeUnit.SECONDS).until(() -> { if (outstandingRequests.size() == 0) { @@ -307,11 +319,13 @@ public PulsarClient answer(InvocationOnMock invocation) throws Throwable { @Test public void testTransactionBufferChannelUnActive() throws PulsarServerException { PulsarService pulsarService = pulsarServiceList.get(0); - PulsarClient mockClient = mock(PulsarClientImpl.class); + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); CompletableFuture completableFuture = new CompletableFuture<>(); ClientCnx clientCnx = mock(ClientCnx.class); completableFuture.complete(clientCnx); - when(((PulsarClientImpl)mockClient).getConnection(anyString())).thenReturn(completableFuture); + when(((PulsarClientImpl)mockClient).getConnection(anyString(), anyInt())).thenReturn(completableFuture); ChannelHandlerContext cnx = mock(ChannelHandlerContext.class); when(clientCnx.ctx()).thenReturn(cnx); Channel channel = mock(Channel.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java index d6ec092c4456f..278cdbac1f09d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferHandlerImplTest.java @@ -24,6 +24,7 @@ import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferHandlerImpl; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -31,8 +32,8 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.api.proto.TxnAction; import org.apache.pulsar.common.naming.NamespaceBundle; @@ -46,7 +47,9 @@ public class TransactionBufferHandlerImplTest { @Test public void testRequestCredits() throws PulsarServerException { - PulsarClient pulsarClient = mock(PulsarClientImpl.class); + PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); PulsarService pulsarService = mock(PulsarService.class); NamespaceService namespaceService = mock(NamespaceService.class); when(pulsarService.getNamespaceService()).thenReturn(namespaceService); @@ -54,7 +57,10 @@ public void testRequestCredits() throws PulsarServerException { when(namespaceService.getBundleAsync(any())).thenReturn(CompletableFuture.completedFuture(mock(NamespaceBundle.class))); Optional opData = Optional.empty(); when(namespaceService.getOwnerAsync(any())).thenReturn(CompletableFuture.completedFuture(opData)); - when(((PulsarClientImpl)pulsarClient).getConnection(anyString())).thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); + when(((PulsarClientImpl)pulsarClient).getConnection(anyString(), anyInt())) + .thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); + when(((PulsarClientImpl)pulsarClient).getConnection(anyString())) + .thenReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))); TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(pulsarService, null, 1000, 3000)); doNothing().when(handler).endTxn(any()); doReturn(CompletableFuture.completedFuture(mock(ClientCnx.class))).when(handler).getClientCnx(anyString()); @@ -75,7 +81,9 @@ public void testRequestCredits() throws PulsarServerException { @Test public void testMinRequestCredits() throws PulsarServerException { - PulsarClient pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); PulsarService pulsarService = mock(PulsarService.class); when(pulsarService.getClient()).thenReturn(pulsarClient); TransactionBufferHandlerImpl handler = spy(new TransactionBufferHandlerImpl(pulsarService, null, 50, 3000)); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java index fb564bd5083c1..e7613879d5152 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java @@ -31,9 +31,13 @@ import java.util.function.Supplier; import java.util.stream.IntStream; import io.netty.util.concurrent.Promise; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; +import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterClass; @@ -80,6 +84,36 @@ public void testSingleIpAddress() throws Exception { eventLoop.shutdownGracefully(); } + @Test + public void testSelectConnectionForSameProducer() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://sample/standalone/ns/tp_"); + admin.topics().createNonPartitionedTopic(topicName); + final CommandCloseProducer commandCloseProducer = new CommandCloseProducer(); + // 10 connection per broker. + final PulsarClient clientWith10ConPerBroker = PulsarClient.builder().connectionsPerBroker(10) + .serviceUrl(lookupUrl.toString()).build(); + ProducerImpl producer = (ProducerImpl) clientWith10ConPerBroker.newProducer().topic(topicName).create(); + commandCloseProducer.setProducerId(producer.producerId); + // An error will be reported when the Producer reconnects using a different connection. + // If no error is reported, the same connection was used when reconnecting. + for (int i = 0; i < 20; i++) { + // Trigger reconnect + ClientCnx cnx = producer.getClientCnx(); + if (cnx != null) { + cnx.handleCloseProducer(commandCloseProducer); + Awaitility.await().untilAsserted(() -> + Assert.assertEquals(producer.getState().toString(), HandlerState.State.Ready.toString(), + "The producer uses a different connection when reconnecting") + ); + } + } + + // cleanup. + producer.close(); + clientWith10ConPerBroker.close(); + admin.topics().delete(topicName); + } + @Test public void testDoubleIpAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); @@ -200,14 +234,16 @@ protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws ClientCnx cnx = pool.getConnection( InetSocketAddress.createUnresolved("proxy", 9999), - InetSocketAddress.createUnresolved("proxy", 9999)).get(); + InetSocketAddress.createUnresolved("proxy", 9999), + pool.genRandomKeyToSelectCon()).get(); Assert.assertEquals(cnx.remoteHostName, "proxy"); Assert.assertNull(cnx.proxyToTargetBrokerAddress); cnx.close(); cnx = pool.getConnection( InetSocketAddress.createUnresolved("broker", 9999), - InetSocketAddress.createUnresolved("proxy", 9999)).get(); + InetSocketAddress.createUnresolved("proxy", 9999), + pool.genRandomKeyToSelectCon()).get(); Assert.assertEquals(cnx.remoteHostName, "proxy"); Assert.assertEquals(cnx.proxyToTargetBrokerAddress, "broker:9999"); cnx.close(); @@ -215,7 +251,8 @@ protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws cnx = pool.getConnection( InetSocketAddress.createUnresolved("broker", 9999), - InetSocketAddress.createUnresolved("broker", 9999)).get(); + InetSocketAddress.createUnresolved("broker", 9999), + pool.genRandomKeyToSelectCon()).get(); Assert.assertEquals(cnx.remoteHostName, "broker"); Assert.assertNull(cnx.proxyToTargetBrokerAddress); cnx.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java index 0ba8511c0ca74..4f7ac6e8de942 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java @@ -122,7 +122,7 @@ public CompletableFuture getConnection(String topic) { result.completeExceptionally(new IOException("New connections are rejected.")); return result; } else { - return super.getConnection(topic); + return super.getConnection(topic, getCnxPool().genRandomKeyToSelectCon()); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java index 5f83ef97d32ca..464f05186fa30 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.compaction; import static org.apache.pulsar.client.impl.RawReaderTest.extractKey; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -48,6 +50,7 @@ import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawMessageImpl; import org.apache.pulsar.common.policies.data.ClusterData; @@ -258,7 +261,10 @@ public void testCompactEmptyTopic() throws Exception { public void testPhaseOneLoopTimeConfiguration() { ServiceConfiguration configuration = new ServiceConfiguration(); configuration.setBrokerServiceCompactionPhaseOneLoopTimeInSeconds(60); - TwoPhaseCompactor compactor = new TwoPhaseCompactor(configuration, Mockito.mock(PulsarClientImpl.class), + PulsarClientImpl mockClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); + TwoPhaseCompactor compactor = new TwoPhaseCompactor(configuration, mockClient, Mockito.mock(BookKeeper.class), compactionScheduler); Assert.assertEquals(compactor.getPhaseOneLoopReadTimeoutInSeconds(), 60); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index 263507dac1dc6..fc7c89c3ce693 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -43,6 +43,7 @@ public class ConnectionHandler { private volatile long epoch = -1L; protected volatile long lastConnectionClosedTimestamp = 0L; private final AtomicBoolean duringConnect = new AtomicBoolean(false); + protected final int randomKeyForSelectConnection; interface Connection { @@ -58,6 +59,7 @@ default void connectionFailed(PulsarClientException e) { protected ConnectionHandler(HandlerState state, Backoff backoff, Connection connection) { this.state = state; + this.randomKeyForSelectConnection = state.client.getCnxPool().genRandomKeyToSelectCon(); this.connection = connection; this.backoff = backoff; CLIENT_CNX_UPDATER.set(this, null); @@ -88,11 +90,11 @@ protected void grabCnx() { if (state.redirectedClusterURI != null) { InetSocketAddress address = InetSocketAddress.createUnresolved(state.redirectedClusterURI.getHost(), state.redirectedClusterURI.getPort()); - cnxFuture = state.client.getConnection(address, address); + cnxFuture = state.client.getConnection(address, address, randomKeyForSelectConnection); } else if (state.topic == null) { cnxFuture = state.client.getConnectionToServiceUrl(); } else { - cnxFuture = state.client.getConnection(state.topic); // + cnxFuture = state.client.getConnection(state.topic, randomKeyForSelectConnection); } cnxFuture.thenCompose(cnx -> connection.connectionOpened(cnx)) .thenAccept(__ -> duringConnect.set(false)) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 1420d81c688ee..ef3a9249c820a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -165,8 +165,18 @@ private static AddressResolver createAddressResolver(ClientCo private static final Random random = new Random(); + public int genRandomKeyToSelectCon() { + if (maxConnectionsPerHosts == 0) { + return -1; + } + return signSafeMod(random.nextInt(), maxConnectionsPerHosts); + } + public CompletableFuture getConnection(final InetSocketAddress address) { - return getConnection(address, address); + if (maxConnectionsPerHosts == 0) { + return getConnection(address, address, -1); + } + return getConnection(address, address, signSafeMod(random.nextInt(), maxConnectionsPerHosts)); } void closeAllConnections() { @@ -204,14 +214,12 @@ void closeAllConnections() { * @return a future that will produce the ClientCnx object */ public CompletableFuture getConnection(InetSocketAddress logicalAddress, - InetSocketAddress physicalAddress) { + InetSocketAddress physicalAddress, final int randomKey) { if (maxConnectionsPerHosts == 0) { // Disable pooling return createConnection(logicalAddress, physicalAddress, -1); } - final int randomKey = signSafeMod(random.nextInt(), maxConnectionsPerHosts); - final ConcurrentMap> innerPool = pool.computeIfAbsent(logicalAddress, a -> new ConcurrentHashMap<>()); CompletableFuture completableFuture = innerPool diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index 6c749a8cf4354..fdabb5fa8cfa5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -944,10 +944,20 @@ public void updateTlsTrustStorePathAndPassword(String tlsTrustStorePath, String conf.setTlsTrustStorePassword(tlsTrustStorePassword); } + public CompletableFuture getConnection(final String topic, int randomKeyForSelectConnection) { + TopicName topicName = TopicName.get(topic); + return lookup.getBroker(topicName) + .thenCompose(pair -> getConnection(pair.getLeft(), pair.getRight(), randomKeyForSelectConnection)); + } + + /** + * Only for test. + */ + @VisibleForTesting public CompletableFuture getConnection(final String topic) { TopicName topicName = TopicName.get(topic); return lookup.getBroker(topicName) - .thenCompose(pair -> getConnection(pair.getLeft(), pair.getRight())); + .thenCompose(pair -> getConnection(pair.getLeft(), pair.getRight(), cnxPool.genRandomKeyToSelectCon())); } public CompletableFuture getConnectionToServiceUrl() { @@ -956,12 +966,13 @@ public CompletableFuture getConnectionToServiceUrl() { "Can't get client connection to HTTP service URL", null)); } InetSocketAddress address = lookup.resolveHost(); - return getConnection(address, address); + return getConnection(address, address, cnxPool.genRandomKeyToSelectCon()); } public CompletableFuture getConnection(final InetSocketAddress logicalAddress, - final InetSocketAddress physicalAddress) { - return cnxPool.getConnection(logicalAddress, physicalAddress); + final InetSocketAddress physicalAddress, + final int randomKeyForSelectConnection) { + return cnxPool.getConnection(logicalAddress, physicalAddress, randomKeyForSelectConnection); } /** visible for pulsar-functions. **/ diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java index 0418a54c772cc..1d1a6f85bfd41 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AcknowledgementsGroupingTrackerTest.java @@ -65,6 +65,8 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { ConcurrentOpenHashMap.newBuilder().build(); cnx = spy(new ClientCnxTest(new ClientConfigurationData(), eventLoopGroup)); PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); doReturn(client).when(consumer).getClient(); doReturn(cnx).when(consumer).getClientCnx(); doReturn(new ConsumerStatsRecorderImpl()).when(consumer).getStats(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java index 36ffa30296bb0..63fbb239439bd 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/AutoClusterFailoverTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -121,6 +122,8 @@ public void testInitialize() { AutoClusterFailover autoClusterFailover = Mockito.spy((AutoClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); Mockito.doReturn(false).when(autoClusterFailover).probeAvailable(primary); Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(secondary); Mockito.doReturn(configurationData).when(pulsarClient).getConfiguration(); @@ -163,6 +166,8 @@ public void testAutoClusterFailoverSwitchWithoutAuthentication() { AutoClusterFailover autoClusterFailover = Mockito.spy((AutoClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); Mockito.doReturn(false).when(autoClusterFailover).probeAvailable(primary); Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(secondary); Mockito.doReturn(configurationData).when(pulsarClient).getConfiguration(); @@ -217,6 +222,8 @@ public void testAutoClusterFailoverSwitchWithAuthentication() throws IOException AutoClusterFailover autoClusterFailover = Mockito.spy((AutoClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); Mockito.doReturn(false).when(autoClusterFailover).probeAvailable(primary); Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(secondary); Mockito.doReturn(configurationData).when(pulsarClient).getConfiguration(); @@ -270,6 +277,8 @@ public void testAutoClusterFailoverSwitchTlsTrustStore() throws IOException { AutoClusterFailover autoClusterFailover = Mockito.spy((AutoClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); Mockito.doReturn(false).when(autoClusterFailover).probeAvailable(primary); Mockito.doReturn(true).when(autoClusterFailover).probeAvailable(secondary); Mockito.doReturn(configurationData).when(pulsarClient).getConfiguration(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java index 4b80e19c256d7..abb195c9830d0 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageContainerImplTest.java @@ -105,6 +105,8 @@ public void recoveryAfterOom() { final ProducerConfigurationData producerConfigurationData = new ProducerConfigurationData(); producerConfigurationData.setCompressionType(CompressionType.NONE); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); MemoryLimitController memoryLimitController = mock(MemoryLimitController.class); when(pulsarClient.getMemoryLimitController()).thenReturn(memoryLimitController); try { @@ -148,6 +150,8 @@ public void testMessagesSize() throws Exception { final ProducerConfigurationData producerConfigurationData = new ProducerConfigurationData(); producerConfigurationData.setCompressionType(CompressionType.NONE); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); MemoryLimitController memoryLimitController = mock(MemoryLimitController.class); when(pulsarClient.getMemoryLimitController()).thenReturn(memoryLimitController); try { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java index 4db5dbe877685..ff7d7f12dd452 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientTestFixtures.java @@ -19,7 +19,9 @@ package org.apache.pulsar.client.impl; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import io.netty.channel.ChannelHandlerContext; @@ -72,11 +74,16 @@ static PulsarClientImpl mockClientCnx(PulsarClientImpl clientMock) { .thenReturn(CompletableFuture.completedFuture(mock(ProducerResponse.class))); when(clientCnxMock.channel().remoteAddress()).thenReturn(mock(SocketAddress.class)); when(clientMock.getConnection(any())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); - when(clientMock.getConnection(any(), any())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); + when(clientMock.getConnection(anyString())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); + when(clientMock.getConnection(anyString(), anyInt())) + .thenReturn(CompletableFuture.completedFuture(clientCnxMock)); + when(clientMock.getConnection(any(), any(), anyInt())) + .thenReturn(CompletableFuture.completedFuture(clientCnxMock)); ConnectionPool connectionPoolMock = mock(ConnectionPool.class); when(clientMock.getCnxPool()).thenReturn(connectionPoolMock); when(connectionPoolMock.getConnection(any())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); - when(connectionPoolMock.getConnection(any(), any())).thenReturn(CompletableFuture.completedFuture(clientCnxMock)); + when(connectionPoolMock.getConnection(any(), any(), anyInt())) + .thenReturn(CompletableFuture.completedFuture(clientCnxMock)); return clientMock; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java index 8dbd23f9c29c9..3fe136630462f 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ConsumerBuilderImplTest.java @@ -76,6 +76,8 @@ public class ConsumerBuilderImplTest { @BeforeMethod(alwaysRun = true) public void setup() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); ConsumerConfigurationData consumerConfigurationData = mock(ConsumerConfigurationData.class); when(consumerConfigurationData.getTopicsPattern()).thenReturn(Pattern.compile("\\w+")); when(consumerConfigurationData.getSubscriptionName()).thenReturn("testSubscriptionName"); @@ -104,6 +106,8 @@ public void testConsumerBuilderImpl() throws PulsarClientException { @Test(expectedExceptions = IllegalArgumentException.class) public void testConsumerBuilderImplWhenSchemaIsNull() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); ConsumerConfigurationData consumerConfigurationData = mock(ConsumerConfigurationData.class); new ConsumerBuilderImpl(client, consumerConfigurationData, null); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java index 570b139832806..227e0db10b724 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ControlledClusterFailoverTest.java @@ -31,6 +31,7 @@ import org.testng.annotations.Test; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @Test(groups = "broker-impl") public class ControlledClusterFailoverTest { @@ -88,6 +89,8 @@ public void testControlledClusterFailoverSwitch() throws IOException { ControlledClusterFailover controlledClusterFailover = Mockito.spy((ControlledClusterFailover) provider); PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); controlledClusterFailover.initialize(pulsarClient); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java index 223881d85a87b..2bd18f69386f1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PartitionedProducerImplTest.java @@ -70,6 +70,8 @@ public class PartitionedProducerImplTest { @BeforeMethod(alwaysRun = true) public void setup() { client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); schema = mock(Schema.class); producerInterceptors = mock(ProducerInterceptors.class); producerCreatedFuture = new CompletableFuture<>(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java index bb3e3fc3accf6..b830d375303bb 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerBuilderImplTest.java @@ -52,6 +52,8 @@ public class ProducerBuilderImplTest { public void setup() { Producer producer = mock(Producer.class); client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); producerBuilderImpl = new ProducerBuilderImpl<>(client, Schema.BYTES); when(client.newProducer()).thenReturn(producerBuilderImpl); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java index 32d0eff6e792e..27e2dcb37cee0 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ProducerStatsRecorderImplTest.java @@ -40,6 +40,8 @@ public void testIncrementNumAcksReceived() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setStatsIntervalSeconds(1); PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.getConfiguration()).thenReturn(conf); Timer timer = new HashedWheelTimer(); when(client.timer()).thenReturn(timer); @@ -60,6 +62,8 @@ public void testGetStatsAndCancelStatsTimeoutWithoutArriveUpdateInterval() { ClientConfigurationData conf = new ClientConfigurationData(); conf.setStatsIntervalSeconds(60); PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.getConfiguration()).thenReturn(conf); Timer timer = new HashedWheelTimer(); when(client.timer()).thenReturn(timer); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java index 54d13538d7867..e0b25db891247 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; @@ -122,7 +123,7 @@ public void testConsumerIsClosed() throws Exception { when(cnx.ctx()).thenReturn(ctx); when(cnx.sendRequestWithId(any(ByteBuf.class), anyLong())) .thenReturn(CompletableFuture.completedFuture(mock(ProducerResponse.class))); - when(pool.getConnection(any(InetSocketAddress.class), any(InetSocketAddress.class))) + when(pool.getConnection(any(InetSocketAddress.class), any(InetSocketAddress.class), anyInt())) .thenReturn(CompletableFuture.completedFuture(cnx)); ClientConfigurationData conf = new ClientConfigurationData(); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java index 9959a2038555c..eee8ba4e8f41a 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewBuilderImplTest.java @@ -52,6 +52,8 @@ public void setup() { Reader reader = mock(Reader.class); when(reader.readNextAsync()).thenReturn(CompletableFuture.allOf()); client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.newReader(any(Schema.class))) .thenReturn(new ReaderBuilderImpl(client, Schema.BYTES)); when(client.createReaderAsync(any(ReaderConfigurationData.class), any(Schema.class))) diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java index 68c886bc7211a..6a866034ddbf8 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TableViewImplTest.java @@ -38,6 +38,8 @@ public class TableViewImplTest { @BeforeClass(alwaysRun = true) public void setup() { client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.newReader(any(Schema.class))) .thenReturn(new ReaderBuilderImpl(client, Schema.BYTES)); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java index e462bb4d62cb5..1b39448fbe770 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/TopicListWatcherTest.java @@ -28,7 +28,9 @@ import org.apache.pulsar.common.api.proto.CommandWatchTopicUpdate; import org.apache.pulsar.common.naming.NamespaceName; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -52,6 +54,9 @@ public class TopicListWatcherTest { public void setup() { listener = mock(TopicsChangedListener.class); client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); + when(connectionPool.genRandomKeyToSelectCon()).thenReturn(0); when(client.getConfiguration()).thenReturn(new ClientConfigurationData()); clientCnxFuture = new CompletableFuture<>(); when(client.getConnectionToServiceUrl()).thenReturn(clientCnxFuture); @@ -59,6 +64,9 @@ public void setup() { when(client.timer()).thenReturn(timer); String topic = "persistent://tenant/ns/topic\\d+"; when(client.getConnection(topic)).thenReturn(clientCnxFuture); + when(client.getConnection(topic, 0)).thenReturn(clientCnxFuture); + when(client.getConnection(any(), any(), anyInt())).thenReturn(clientCnxFuture); + when(connectionPool.getConnection(any(), any(), anyInt())).thenReturn(clientCnxFuture); watcherFuture = new CompletableFuture<>(); watcher = new TopicListWatcher(listener, client, Pattern.compile(topic), 7, @@ -67,7 +75,7 @@ public void setup() { @Test public void testWatcherGrabsConnection() { - verify(client).getConnection(any()); + verify(client).getConnection(anyString(), anyInt()); } @Test diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java index f6c668703d9db..91ad321048226 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/UnAckedMessageTrackerTest.java @@ -45,6 +45,8 @@ public class UnAckedMessageTrackerTest { @Test public void testAddAndRemove() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), 1, TimeUnit.MILLISECONDS); when(client.timer()).thenReturn(timer); @@ -83,6 +85,8 @@ public void testAddAndRemove() { @Test public void testTrackChunkedMessageId() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); Timer timer = new HashedWheelTimer(new DefaultThreadFactory("pulsar-timer", Thread.currentThread().isDaemon()), 1, TimeUnit.MILLISECONDS); when(client.timer()).thenReturn(timer); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProviderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProviderTest.java index 8959e67023463..bfd6af37e3ea6 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProviderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/generic/MultiVersionSchemaInfoProviderTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.CompletableFuture; import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.LookupService; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AvroSchema; @@ -46,6 +47,8 @@ public class MultiVersionSchemaInfoProviderTest { @BeforeMethod public void setup() { PulsarClientImpl client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.getLookup()).thenReturn(mock(LookupService.class)); schemaProvider = new MultiVersionSchemaInfoProvider( TopicName.get("persistent://public/default/my-topic"), client); diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java index e0ebb52da7490..90f7df37fa196 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/ContextImplTest.java @@ -47,6 +47,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.MultiTopicsConsumerImpl; import org.apache.pulsar.client.impl.ProducerBase; @@ -99,6 +100,8 @@ public void setup() throws PulsarClientException { producer = mock(Producer.class); client = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(client.getCnxPool()).thenReturn(connectionPool); when(client.newProducer()).thenReturn(new ProducerBuilderImpl(client, Schema.BYTES)); when(client.createProducerAsync(any(ProducerConfigurationData.class), any(), any())) .thenReturn(CompletableFuture.completedFuture(producer)); diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java index fdac39512cc24..799bad839a451 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/sink/PulsarSinkTest.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNull; @@ -56,6 +57,7 @@ import org.apache.pulsar.client.api.schema.GenericSchema; import org.apache.pulsar.client.api.schema.RecordSchemaBuilder; import org.apache.pulsar.client.api.schema.SchemaBuilder; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.schema.AutoConsumeSchema; import org.apache.pulsar.common.functions.FunctionConfig; @@ -95,6 +97,8 @@ public byte[] serialize(String input) { */ private static PulsarClientImpl getPulsarClient() throws PulsarClientException { PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); ConsumerBuilder consumerBuilder = mock(ConsumerBuilder.class); doReturn(consumerBuilder).when(consumerBuilder).topics(anyList()); doReturn(consumerBuilder).when(consumerBuilder).subscriptionName(anyString()); diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/PulsarSourceTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/PulsarSourceTest.java index 91e4c06fe5b49..5d6e4a3dc75e7 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/PulsarSourceTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/source/PulsarSourceTest.java @@ -21,6 +21,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertSame; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertTrue; @@ -44,6 +46,7 @@ import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.api.schema.GenericRecord; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.MessageImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.functions.ConsumerConfig; @@ -105,6 +108,8 @@ public static Object[] getPulsarSourceImpls() { */ private static PulsarClientImpl getPulsarClient() throws PulsarClientException { PulsarClientImpl pulsarClient = Mockito.mock(PulsarClientImpl.class); + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(pulsarClient.getCnxPool()).thenReturn(connectionPool); ConsumerBuilder goodConsumerBuilder = Mockito.mock(ConsumerBuilder.class); ConsumerBuilder badConsumerBuilder = Mockito.mock(ConsumerBuilder.class); Mockito.doReturn(goodConsumerBuilder).when(goodConsumerBuilder) diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java index 8da24fd1b7250..5c10a59bd1388 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/LeaderServiceTest.java @@ -36,6 +36,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; @@ -76,7 +77,8 @@ public LeaderServiceTest() { @BeforeMethod public void setup() throws PulsarClientException { mockClient = mock(PulsarClientImpl.class); - + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); mockConsumer = mock(ConsumerImpl.class); ConsumerBuilder mockConsumerBuilder = mock(ConsumerBuilder.class); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java index e066bb24e6ef0..ac3176b3135e2 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/MembershipManagerTest.java @@ -41,6 +41,7 @@ import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.ReaderBuilder; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.functions.WorkerInfo; @@ -68,7 +69,8 @@ public MembershipManagerTest() { private static PulsarClient mockPulsarClient() throws PulsarClientException { PulsarClientImpl mockClient = mock(PulsarClientImpl.class); - + ConnectionPool connectionPool = mock(ConnectionPool.class); + when(mockClient.getCnxPool()).thenReturn(connectionPool); ConsumerImpl mockConsumer = mock(ConsumerImpl.class); ConsumerBuilder mockConsumerBuilder = mock(ConsumerBuilder.class); From 597ba821cdce3fdfa1a8c2b177e46ffeca0007d8 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 13 Sep 2023 21:19:42 -0500 Subject: [PATCH 280/494] [fix][ci] Fix missing binary license (#21176) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 9476793d869c1..f622e89a68882 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -436,6 +436,8 @@ The Apache Software License, Version 2.0 - io.grpc-grpc-xds-1.55.3.jar - io.grpc-grpc-rls-1.55.3.jar - com.google.auto.service-auto-service-annotations-1.0.jar + - io.grpc-grpc-servlet-1.55.3.jar + - io.grpc-grpc-servlet-jakarta-1.55.3.jar * Perfmark - io.perfmark-perfmark-api-0.26.0.jar * OpenCensus From ad5bb050cb131eee3f03baf396d216d2d56cabf9 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 14 Sep 2023 00:17:10 +0800 Subject: [PATCH 281/494] [fix] [broker] Make specified producer could override the previous one (#21155) The client assumed the connection was inactive, but the Broker assumed the connection was fine. The Client tried to use a new connection to reconnect a producer, then got an error `Producer with name 'st-0-5' is already connected to topic`. - In a connection, the second connection waits for the first connection to complete\. But there is a bug that causes this mechanism to fail\. - If a producer uses a default name, the second registration will override the first one. But it can not override the first one if it uses a specified producer name\. I think this mechanism is to prevent a client from creating two producers with the same name. However, method `Producer.isSuccessorTo` has checked the `producer-id`, and the `producer-id` of multiple producers created by the same client are different. So this mechanism can be deleted. - For `issue 1`: If a producer with the same name tries to use a new connection, async checks the old connection is available. The producers related to the connection that is not available are automatically cleaned up. - For `issue 2`: - Fix the bug that causes a complete producer future will be removed from `ServerCnx`. - Remove the mechanism that prevents a producer with a specified name from overriding the previous producer. (cherry picked from commit bda16b6f5b715942f7ed996052f6cbd8026fbbf0) --- .../pulsar/broker/service/AbstractTopic.java | 8 +- .../pulsar/broker/service/ServerCnx.java | 40 ++--- .../auth/MockedPulsarServiceBaseTest.java | 28 ++- .../pulsar/broker/service/ServerCnxTest.java | 162 +++++++++++++++++- .../impl/ProducerConsumerInternalTest.java | 147 ++++++++++++++++ 5 files changed, 360 insertions(+), 25 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 97966828fb117..03c6233a9250c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -982,8 +982,7 @@ protected void internalAddProducer(Producer producer) throws BrokerServiceExcept private void tryOverwriteOldProducer(Producer oldProducer, Producer newProducer) throws BrokerServiceException { - if (newProducer.isSuccessorTo(oldProducer) && !isUserProvidedProducerName(oldProducer) - && !isUserProvidedProducerName(newProducer)) { + if (newProducer.isSuccessorTo(oldProducer)) { oldProducer.close(false); if (!producers.replace(newProducer.getProducerName(), oldProducer, newProducer)) { // Met concurrent update, throw exception here so that client can try reconnect later. @@ -993,6 +992,11 @@ private void tryOverwriteOldProducer(Producer oldProducer, Producer newProducer) handleProducerRemoved(oldProducer); } } else { + // If a producer with the same name tries to use a new connection, async check the old connection is + // available. The producers related the connection that not available are automatically cleaned up. + if (!Objects.equals(oldProducer.getCnx(), newProducer.getCnx())) { + oldProducer.getCnx().checkConnectionLiveness(); + } throw new BrokerServiceException.NamingException( "Producer with name '" + newProducer.getProducerName() + "' is already connected to topic"); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index b566ed3a05fc1..3fc1e96a57d96 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -1389,36 +1389,36 @@ protected void handleProducer(final CommandProducer cmdProducer) { CompletableFuture existingProducerFuture = producers.putIfAbsent(producerId, producerFuture); if (existingProducerFuture != null) { - if (existingProducerFuture.isDone() && !existingProducerFuture.isCompletedExceptionally()) { - Producer producer = existingProducerFuture.getNow(null); - log.info("[{}] Producer with the same id is already created:" - + " producerId={}, producer={}", remoteAddress, producerId, producer); - commandSender.sendProducerSuccessResponse(requestId, producer.getProducerName(), - producer.getSchemaVersion()); - return null; - } else { + if (!existingProducerFuture.isDone()) { // There was an early request to create a producer with same producerId. // This can happen when client timeout is lower than the broker timeouts. // We need to wait until the previous producer creation request // either complete or fails. - ServerError error = null; - if (!existingProducerFuture.isDone()) { - error = ServerError.ServiceNotReady; - } else { - error = getErrorCode(existingProducerFuture); - // remove producer with producerId as it's already completed with exception - producers.remove(producerId, existingProducerFuture); - } log.warn("[{}][{}] Producer with id is already present on the connection, producerId={}", remoteAddress, topicName, producerId); - commandSender.sendErrorResponse(requestId, error, "Producer is already present on the connection"); - return null; + commandSender.sendErrorResponse(requestId, ServerError.ServiceNotReady, + "Producer is already present on the connection"); + } else if (existingProducerFuture.isCompletedExceptionally()) { + // remove producer with producerId as it's already completed with exception + log.warn("[{}][{}] Producer with id is failed to register present on the connection, producerId={}", + remoteAddress, topicName, producerId); + ServerError error = getErrorCode(existingProducerFuture); + producers.remove(producerId, existingProducerFuture); + commandSender.sendErrorResponse(requestId, error, + "Producer is already failed to register present on the connection"); + } else { + Producer producer = existingProducerFuture.getNow(null); + log.info("[{}] [{}] Producer with the same id is already created:" + + " producerId={}, producer={}", remoteAddress, topicName, producerId, producer); + commandSender.sendProducerSuccessResponse(requestId, producer.getProducerName(), + producer.getSchemaVersion()); } + return null; } if (log.isDebugEnabled()) { - log.debug("[{}][{}] Creating producer. producerId={}, schema is {}", remoteAddress, topicName, - producerId, schema == null ? "absent" : "present"); + log.debug("[{}][{}] Creating producer. producerId={}, producerName={}, schema is {}", remoteAddress, + topicName, producerId, producerName, schema == null ? "absent" : "present"); } service.getOrCreateTopic(topicName.toString()).thenCompose((Topic topic) -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 3fe8c22b1c4de..b16dc27e26586 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -18,7 +18,8 @@ */ package org.apache.pulsar.broker.auth; -import static org.apache.pulsar.broker.BrokerTestUtil.*; +import static org.apache.pulsar.broker.BrokerTestUtil.spyWithoutRecordingInvocations; +import static org.testng.Assert.assertEquals; import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.net.InetSocketAddress; @@ -37,10 +38,13 @@ import java.util.function.Predicate; import javax.ws.rs.container.AsyncResponse; import javax.ws.rs.container.TimeoutHandler; +import lombok.AllArgsConstructor; +import lombok.Data; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; @@ -48,6 +52,7 @@ import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -55,11 +60,12 @@ import org.apache.pulsar.tests.TestRetrySupport; import org.apache.pulsar.utils.ResourceUtils; import org.apache.zookeeper.MockZooKeeper; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.mockito.Mockito; import org.mockito.internal.util.MockUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.org.awaitility.Awaitility; import org.testng.annotations.DataProvider; /** @@ -623,5 +629,23 @@ public Object[][] incorrectPersistentPolicies() { }; } + protected ServiceProducer getServiceProducer(ProducerImpl clientProducer, String topicName) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + org.apache.pulsar.broker.service.Producer serviceProducer = + persistentTopic.getProducers().get(clientProducer.getProducerName()); + long clientProducerId = WhiteboxImpl.getInternalState(clientProducer, "producerId"); + assertEquals(serviceProducer.getProducerId(), clientProducerId); + assertEquals(serviceProducer.getEpoch(), clientProducer.getConnectionHandler().getEpoch()); + return new ServiceProducer(serviceProducer, persistentTopic); + } + + @Data + @AllArgsConstructor + public static class ServiceProducer { + private org.apache.pulsar.broker.service.Producer serviceProducer; + private PersistentTopic persistentTopic; + } + private static final Logger log = LoggerFactory.getLogger(MockedPulsarServiceBaseTest.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index c3bab634a42c1..2ea5e28880bf8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -47,6 +47,8 @@ import io.netty.channel.ChannelHandler; import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.vertx.core.impl.ConcurrentHashSet; +import java.io.Closeable; import java.io.IOException; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; @@ -64,6 +66,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback; @@ -74,6 +77,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.impl.PositionImpl; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockAlwaysExpiredAuthenticationProvider; @@ -93,6 +97,7 @@ import org.apache.pulsar.broker.service.ServerCnx.State; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.utils.ClientChannelHelper; +import org.apache.pulsar.client.api.ProducerAccessMode; import org.apache.pulsar.client.api.transaction.TxnID; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.api.proto.AuthMethod; @@ -113,6 +118,7 @@ import org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespaceResponse; import org.apache.pulsar.common.api.proto.CommandLookupTopicResponse; import org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadataResponse; +import org.apache.pulsar.common.api.proto.CommandPing; import org.apache.pulsar.common.api.proto.CommandProducerSuccess; import org.apache.pulsar.common.api.proto.CommandSendError; import org.apache.pulsar.common.api.proto.CommandSendReceipt; @@ -135,6 +141,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.Commands.ChecksumType; import org.apache.pulsar.common.protocol.PulsarHandler; +import org.apache.pulsar.common.protocol.schema.EmptyVersion; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; @@ -149,6 +156,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +@Slf4j @SuppressWarnings("unchecked") @Test(groups = "broker") public class ServerCnxTest { @@ -184,10 +192,12 @@ public class ServerCnxTest { private ManagedLedger ledgerMock; private ManagedCursor cursorMock; + private ConcurrentHashSet channelsStoppedAnswerHealthCheck = new ConcurrentHashSet<>(); @BeforeMethod(alwaysRun = true) public void setup() throws Exception { + channelsStoppedAnswerHealthCheck.clear(); svcConfig = new ServiceConfiguration(); svcConfig.setBrokerShutdownTimeoutMs(0L); svcConfig.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); @@ -927,6 +937,134 @@ public void testVerifyOriginalPrincipalWithAuthDataForwardedFromProxy() throws E })); } + @Test + public void testHandleProducerAfterClientChannelInactive() throws Exception { + final String tName = successTopicName; + final long producerId = 1; + final MutableInt requestId = new MutableInt(1); + final MutableInt epoch = new MutableInt(1); + final Map metadata = Collections.emptyMap(); + final String pName = "p1"; + resetChannel(); + setChannelConnected(); + + // The producer register using the first connection. + ByteBuf cmdProducer1 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + channel.writeInbound(cmdProducer1); + assertTrue(getResponse() instanceof CommandProducerSuccess); + PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(tName).get(); + assertNotNull(topicRef); + assertEquals(topicRef.getProducers().size(), 1); + + // Verify the second producer using a new connection will override the producer who using a stopped channel. + channelsStoppedAnswerHealthCheck.add(channel); + ClientChannel channel2 = new ClientChannel(); + setChannelConnected(channel2.serverCnx); + Awaitility.await().untilAsserted(() -> { + ByteBuf cmdProducer2 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + channel2.channel.writeInbound(cmdProducer2); + assertTrue(getResponse(channel2.channel, channel2.clientChannelHelper) instanceof CommandProducerSuccess); + assertEquals(topicRef.getProducers().size(), 1); + }); + + // cleanup. + channel.finish(); + channel2.close(); + } + + private class ClientChannel implements Closeable { + private ClientChannelHelper clientChannelHelper = new ClientChannelHelper(); + private ServerCnx serverCnx = new ServerCnx(pulsar); + private EmbeddedChannel channel = new EmbeddedChannel(new LengthFieldBasedFrameDecoder( + 5 * 1024 * 1024, + 0, + 4, + 0, + 4), + serverCnx); + public ClientChannel() { + serverCnx.setAuthRole(""); + } + public void close(){ + if (channel != null && channel.isActive()) { + serverCnx.close(); + channel.close(); + } + } + } + + @Test + public void testHandleProducer() throws Exception { + final String tName = "persistent://public/default/test-topic"; + final long producerId = 1; + final MutableInt requestId = new MutableInt(1); + final MutableInt epoch = new MutableInt(1); + final Map metadata = Collections.emptyMap(); + final String pName = "p1"; + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + // connect. + ByteBuf cConnect = Commands.newConnect("none", "", null); + channel.writeInbound(cConnect); + assertEquals(serverCnx.getState(), State.Connected); + assertTrue(getResponse() instanceof CommandConnected); + + // There is an in-progress producer registration. + ByteBuf cProducer1 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + CompletableFuture existingFuture1 = new CompletableFuture(); + serverCnx.getProducers().put(producerId, existingFuture1); + channel.writeInbound(cProducer1); + Object response1 = getResponse(); + assertTrue(response1 instanceof CommandError); + CommandError error1 = (CommandError) response1; + assertEquals(error1.getError().toString(), ServerError.ServiceNotReady.toString()); + assertTrue(error1.getMessage().contains("already present on the connection")); + + // There is a failed registration. + ByteBuf cProducer2 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + CompletableFuture existingFuture2 = new CompletableFuture(); + existingFuture2.completeExceptionally(new BrokerServiceException.ProducerBusyException("123")); + serverCnx.getProducers().put(producerId, existingFuture2); + + channel.writeInbound(cProducer2); + Object response2 = getResponse(); + assertTrue(response2 instanceof CommandError); + CommandError error2 = (CommandError) response2; + assertEquals(error2.getError().toString(), ServerError.ProducerBusy.toString()); + assertTrue(error2.getMessage().contains("already failed to register present on the connection")); + + // There is an successful registration. + ByteBuf cProducer3 = Commands.newProducer(tName, producerId, requestId.incrementAndGet(), + pName, false, metadata, null, epoch.incrementAndGet(), false, + ProducerAccessMode.Shared, Optional.empty(), false); + CompletableFuture existingFuture3 = new CompletableFuture(); + org.apache.pulsar.broker.service.Producer serviceProducer = + mock(org.apache.pulsar.broker.service.Producer.class); + when(serviceProducer.getProducerName()).thenReturn(pName); + when(serviceProducer.getSchemaVersion()).thenReturn(new EmptyVersion()); + existingFuture3.complete(serviceProducer); + serverCnx.getProducers().put(producerId, existingFuture3); + + channel.writeInbound(cProducer3); + Object response3 = getResponse(); + assertTrue(response3 instanceof CommandProducerSuccess); + CommandProducerSuccess cProducerSuccess = (CommandProducerSuccess) response3; + assertEquals(cProducerSuccess.getProducerName(), pName); + + // cleanup. + channel.finish(); + } + // This test used to be in the ServerCnxAuthorizationTest class, but it was migrated here because the mocking // in that class was too extensive. There is some overlap with this test and other tests in this class. The primary // role of this test is verifying that the correct role and AuthenticationDataSource are passed to the @@ -2471,6 +2609,10 @@ protected void resetChannel() throws Exception { } protected void setChannelConnected() throws Exception { + setChannelConnected(serverCnx); + } + + protected void setChannelConnected(ServerCnx serverCnx) throws Exception { Field channelState = ServerCnx.class.getDeclaredField("state"); channelState.setAccessible(true); channelState.set(serverCnx, State.Connected); @@ -2484,13 +2626,31 @@ private void setConnectionVersion(int version) throws Exception { } protected Object getResponse() throws Exception { + return getResponse(channel, clientChannelHelper); + } + + protected Object getResponse(EmbeddedChannel channel, ClientChannelHelper clientChannelHelper) throws Exception { // Wait at most for 10s to get a response final long sleepTimeMs = 10; final long iterations = TimeUnit.SECONDS.toMillis(10) / sleepTimeMs; for (int i = 0; i < iterations; i++) { if (!channel.outboundMessages().isEmpty()) { Object outObject = channel.outboundMessages().remove(); - return clientChannelHelper.getCommand(outObject); + Object cmd = clientChannelHelper.getCommand(outObject); + if (cmd instanceof CommandPing) { + if (channelsStoppedAnswerHealthCheck.contains(channel)) { + continue; + } + channel.writeAndFlush(Commands.newPong()).addListener(future -> { + if (!future.isSuccess()) { + log.warn("[{}] Forcing connection to close since cannot send a pong message.", + channel, future.cause()); + channel.close(); + } + }); + continue; + } + return cmd; } else { Thread.sleep(sleepTimeMs); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java new file mode 100644 index 0000000000000..f05f735635746 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ProducerConsumerInternalTest.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import java.util.concurrent.CountDownLatch; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.ServerCnx; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.common.api.proto.CommandCloseProducer; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Different with {@link org.apache.pulsar.client.api.SimpleProducerConsumerTest}, this class can visit the variables + * of {@link ConsumerImpl} which are modified `protected`. + */ +@Test(groups = "broker-api") +public class ProducerConsumerInternalTest extends ProducerConsumerBase { + + @BeforeClass(alwaysRun = true) + @Override + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testSameProducerRegisterTwice() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + admin.topics().createNonPartitionedTopic(topicName); + + // Create producer using default producerName. + ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().topic(topicName).create(); + ServiceProducer serviceProducer = getServiceProducer(producer, topicName); + + // Remove producer maintained by server cnx. To make it can register the second time. + removeServiceProducerMaintainedByServerCnx(serviceProducer); + + // Trigger the client producer reconnect. + CommandCloseProducer commandCloseProducer = new CommandCloseProducer(); + commandCloseProducer.setProducerId(producer.producerId); + producer.getClientCnx().handleCloseProducer(commandCloseProducer); + + // Verify the reconnection will be success. + Awaitility.await().untilAsserted(() -> { + assertEquals(producer.getState().toString(), "Ready"); + }); + } + + @Test + public void testSameProducerRegisterTwiceWithSpecifiedProducerName() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String pName = "p1"; + admin.topics().createNonPartitionedTopic(topicName); + + // Create producer using default producerName. + ProducerImpl producer = (ProducerImpl) pulsarClient.newProducer().producerName(pName).topic(topicName).create(); + ServiceProducer serviceProducer = getServiceProducer(producer, topicName); + + // Remove producer maintained by server cnx. To make it can register the second time. + removeServiceProducerMaintainedByServerCnx(serviceProducer); + + // Trigger the client producer reconnect. + CommandCloseProducer commandCloseProducer = new CommandCloseProducer(); + commandCloseProducer.setProducerId(producer.producerId); + producer.getClientCnx().handleCloseProducer(commandCloseProducer); + + // Verify the reconnection will be success. + Awaitility.await().untilAsserted(() -> { + assertEquals(producer.getState().toString(), "Ready", "The producer registration failed"); + }); + } + + private void removeServiceProducerMaintainedByServerCnx(ServiceProducer serviceProducer) { + ServerCnx serverCnx = (ServerCnx) serviceProducer.getServiceProducer().getCnx(); + serverCnx.removedProducer(serviceProducer.getServiceProducer()); + Awaitility.await().untilAsserted(() -> { + assertFalse(serverCnx.getProducers().containsKey(serviceProducer.getServiceProducer().getProducerId())); + }); + } + + @Test + public void testExclusiveConsumerWillAlwaysRetryEvenIfReceivedConsumerBusyError() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String subscriptionName = "subscription1"; + admin.topics().createNonPartitionedTopic(topicName); + + final ConsumerImpl consumer = (ConsumerImpl) pulsarClient.newConsumer().topic(topicName.toString()) + .subscriptionType(SubscriptionType.Exclusive).subscriptionName(subscriptionName).subscribe(); + + ClientCnx clientCnx = consumer.getClientCnx(); + ServerCnx serverCnx = (ServerCnx) pulsar.getBrokerService() + .getTopic(topicName,false).join().get().getSubscription(subscriptionName) + .getDispatcher().getConsumers().get(0).cnx(); + + // Make a disconnect to trigger broker remove the consumer which related this connection. + // Make the second subscribe runs after the broker removing the old consumer, then it will receive + // an error: "Exclusive consumer is already connected" + final CountDownLatch countDownLatch = new CountDownLatch(1); + serverCnx.execute(() -> { + try { + countDownLatch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + clientCnx.close(); + Thread.sleep(1000); + countDownLatch.countDown(); + + // Verify the consumer will always retry subscribe event received ConsumerBusy error. + Awaitility.await().untilAsserted(() -> { + assertEquals(consumer.getState(), HandlerState.State.Ready); + }); + + // cleanup. + consumer.close(); + admin.topics().delete(topicName, false); + } +} From faee1233d8b9dab229d8e4c67fdd28c307da2ed1 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 13 Sep 2023 17:30:57 +0800 Subject: [PATCH 282/494] [improve] [broker] improve read entry error log for troubleshooting (#21169) (cherry picked from commit 65706c6ffa737f946ad9a1bfdb6ff70fa66c0415) --- .../mledger/impl/ManagedCursorImpl.java | 10 ++++++++++ .../mledger/impl/ManagedLedgerImpl.java | 6 ++++++ .../impl/ReadOnlyManagedLedgerImpl.java | 4 ++-- .../admin/impl/PersistentTopicsBase.java | 18 ++++++++++++++++++ .../persistent/PersistentReplicator.java | 6 ++++++ .../persistent/PersistentSubscription.java | 6 ++++++ .../service/persistent/PersistentTopic.java | 2 +- ...SnapshotSegmentAbortedTxnProcessorImpl.java | 6 ++++++ 8 files changed, 55 insertions(+), 3 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 2e0668639398a..3631feccca946 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -816,6 +816,11 @@ public void readEntryComplete(Entry entry, Object ctx) { result.entry = entry; counter.countDown(); } + + @Override + public String toString() { + return String.format("Cursor [{}] get Nth entry", ManagedCursorImpl.this); + } }, null); counter.await(ledger.getConfig().getMetadataOperationsTimeoutSeconds(), TimeUnit.SECONDS); @@ -1528,6 +1533,11 @@ public synchronized void readEntryFailed(ManagedLedgerException mle, Object ctx) callback.readEntriesFailed(exception.get(), ctx); } } + + @Override + public String toString() { + return String.format("Cursor [{}] async replay entries", ManagedCursorImpl.this); + } }; positions.stream().filter(position -> !alreadyAcknowledgedPositions.contains(position)) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 89c18f4b834f3..4991080d38cbe 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1263,6 +1263,12 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { log.error("Error read entry for position {}", nextPos, exception); future.completeExceptionally(exception); } + + @Override + public String toString() { + return String.format("ML [{}] get earliest message publish time of pos", + ManagedLedgerImpl.this.name); + } }, null); return future; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java index 944674f6862c2..1fdf69395068f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyManagedLedgerImpl.java @@ -143,8 +143,8 @@ public void asyncReadEntry(PositionImpl position, AsyncCallbacks.ReadEntryCallba this.getLedgerHandle(position.getLedgerId()) .thenAccept((ledger) -> asyncReadEntry(ledger, position, callback, ctx)) .exceptionally((ex) -> { - log.error("[{}] Error opening ledger for reading at position {} - {}", this.name, position, - ex.getMessage()); + log.error("[{}] Error opening ledger for reading at position {} - {}. Op: {}", this.name, + position, ex.getMessage(), callback); callback.readEntryFailed(ManagedLedgerException.getManagedLedgerException(ex.getCause()), ctx); return null; }); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 81f5e3c1f323b..5a1462abeb7bb 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -2824,6 +2824,12 @@ public void readEntryComplete(Entry entry, Object ctx) { } } } + + @Override + public String toString() { + return String.format("Topic [{}] get entry batch size", + PersistentTopicsBase.this.topicName); + } }, null); } catch (NullPointerException npe) { batchSizeFuture.completeExceptionally(new RestException(Status.NOT_FOUND, "Message not found")); @@ -2922,6 +2928,12 @@ public void readEntryComplete(Entry entry, Object ctx) { } } } + + @Override + public String toString() { + return String.format("Topic [{}] internal get message by id", + PersistentTopicsBase.this.topicName); + } }, null); return results; }); @@ -3088,6 +3100,12 @@ public void readEntryComplete(Entry entry, Object ctx) { public void readEntryFailed(ManagedLedgerException exception, Object ctx) { future.completeExceptionally(exception); } + + @Override + public String toString() { + return String.format("Topic [{}] internal examine message async", + PersistentTopicsBase.this.topicName); + } }, null); return future; } catch (ManagedLedgerException exception) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index ccf70eecec35c..0b3415243399d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -529,6 +529,12 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { public void readEntryComplete(Entry entry, Object ctx) { future.complete(entry); } + + @Override + public String toString() { + return String.format("Replication [{}] peek Nth message", + PersistentReplicator.this.producer.getProducerName()); + } }, null); return future; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 0c77c7662fd35..3487aa9fb2ade 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -852,6 +852,12 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { public void readEntryComplete(Entry entry, Object ctx) { future.complete(entry); } + + @Override + public String toString() { + return String.format("Subscription [{}-{}] async replay entries", PersistentSubscription.this.topicName, + PersistentSubscription.this.subName); + } }, null); return future; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 85c1d98e04736..22459b2f61575 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -3040,7 +3040,7 @@ public boolean isOldestMessageExpired(ManagedCursor cursor, int messageTTLInSeco // if AutoSkipNonRecoverableData is set to true, just return true here. return true; } else { - log.warn("[{}] Error while getting the oldest message", topic, e); + log.warn("[{}] [{}] Error while getting the oldest message", topic, cursor.toString(), e); } } finally { if (entry != null) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index fc92754d3008e..258a253c12e62 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -322,6 +322,12 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { hasInvalidIndex.set(true); } } + + @Override + public String toString() { + return String.format("Transaction buffer [{}] recover from snapshot", + SnapshotSegmentAbortedTxnProcessorImpl.this.topic.getName()); + } }, null); }); openManagedLedgerAndHandleSegmentsFuture.complete(null); From 91544c9cad45b6435dd3a7edaf8fa95bf55a4f5a Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Fri, 15 Sep 2023 10:26:17 +0800 Subject: [PATCH 283/494] [fix][broker] Avoid splitting one batch message into two entries in StrategicTwoPhaseCompactor (#21156) --- .../impl/RawBatchMessageContainerImpl.java | 41 +++++++-- .../StrategicTwoPhaseCompactor.java | 90 ++++++++----------- .../RawBatchMessageContainerImplTest.java | 53 +++++------ .../pulsar/compaction/CompactionTest.java | 11 +-- .../StrategicCompactionRetentionTest.java | 2 +- .../compaction/StrategicCompactionTest.java | 66 +++++++++++++- .../compaction/StrategicCompactorTest.java | 4 +- 7 files changed, 172 insertions(+), 95 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java index 7e1c2cd5e3fe3..ba8d3db7178d9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java @@ -23,6 +23,7 @@ import java.util.Set; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.MessageCrypto; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.crypto.MessageCryptoBc; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; @@ -44,17 +45,17 @@ * [(k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3)] */ public class RawBatchMessageContainerImpl extends BatchMessageContainerImpl { - MessageCrypto msgCrypto; - Set encryptionKeys; - CryptoKeyReader cryptoKeyReader; + private MessageCrypto msgCrypto; + private Set encryptionKeys; + private CryptoKeyReader cryptoKeyReader; + private MessageIdAdv lastAddedMessageId; - public RawBatchMessageContainerImpl(int maxNumMessagesInBatch, int maxBytesInBatch) { + public RawBatchMessageContainerImpl() { super(); this.compressionType = CompressionType.NONE; this.compressor = new CompressionCodecNone(); - this.maxNumMessagesInBatch = maxNumMessagesInBatch; - this.maxBytesInBatch = maxBytesInBatch; } + private ByteBuf encrypt(ByteBuf compressedPayload) { if (msgCrypto == null) { return compressedPayload; @@ -90,6 +91,28 @@ public void setCryptoKeyReader(CryptoKeyReader cryptoKeyReader) { this.cryptoKeyReader = cryptoKeyReader; } + @Override + public boolean add(MessageImpl msg, SendCallback callback) { + this.lastAddedMessageId = (MessageIdAdv) msg.getMessageId(); + return super.add(msg, callback); + } + + @Override + protected boolean isBatchFull() { + return false; + } + + @Override + public boolean haveEnoughSpace(MessageImpl msg) { + if (lastAddedMessageId == null) { + return true; + } + // Keep same batch compact to same batch. + MessageIdAdv msgId = (MessageIdAdv) msg.getMessageId(); + return msgId.getLedgerId() == lastAddedMessageId.getLedgerId() + && msgId.getEntryId() == lastAddedMessageId.getEntryId(); + } + /** * Serializes the batched messages and return the ByteBuf. * It sets the CompressionType and Encryption Keys from the batched messages. @@ -168,4 +191,10 @@ public ByteBuf toByteBuf() { clear(); return buf; } + + @Override + public void clear() { + this.lastAddedMessageId = null; + super.clear(); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java index a6b0942742763..fefa2ee959cc5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/StrategicTwoPhaseCompactor.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.compaction; -import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import java.time.Duration; import java.util.Iterator; @@ -63,39 +62,19 @@ public class StrategicTwoPhaseCompactor extends TwoPhaseCompactor { private static final Logger log = LoggerFactory.getLogger(StrategicTwoPhaseCompactor.class); private static final int MAX_OUTSTANDING = 500; - private static final int MAX_NUM_MESSAGES_IN_BATCH = 1000; - private static final int MAX_BYTES_IN_BATCH = 128 * 1024; private static final int MAX_READER_RECONNECT_WAITING_TIME_IN_MILLIS = 20 * 1000; private final Duration phaseOneLoopReadTimeout; private final RawBatchMessageContainerImpl batchMessageContainer; - @VisibleForTesting public StrategicTwoPhaseCompactor(ServiceConfiguration conf, PulsarClient pulsar, BookKeeper bk, - ScheduledExecutorService scheduler, - int maxNumMessagesInBatch) { - this(conf, pulsar, bk, scheduler, maxNumMessagesInBatch, MAX_BYTES_IN_BATCH); - } - - private StrategicTwoPhaseCompactor(ServiceConfiguration conf, - PulsarClient pulsar, - BookKeeper bk, - ScheduledExecutorService scheduler, - int maxNumMessagesInBatch, - int maxBytesInBatch) { + ScheduledExecutorService scheduler) { super(conf, pulsar, bk, scheduler); - batchMessageContainer = new RawBatchMessageContainerImpl(maxNumMessagesInBatch, maxBytesInBatch); + batchMessageContainer = new RawBatchMessageContainerImpl(); phaseOneLoopReadTimeout = Duration.ofSeconds(conf.getBrokerServiceCompactionPhaseOneLoopTimeInSeconds()); } - public StrategicTwoPhaseCompactor(ServiceConfiguration conf, - PulsarClient pulsar, - BookKeeper bk, - ScheduledExecutorService scheduler) { - this(conf, pulsar, bk, scheduler, MAX_NUM_MESSAGES_IN_BATCH, MAX_BYTES_IN_BATCH); - } - public CompletableFuture compact(String topic) { throw new UnsupportedOperationException(); } @@ -418,7 +397,6 @@ private void phaseTwoLoop(String topic, Iterator> reader, .whenComplete((res, exception2) -> { if (exception2 != null) { promise.completeExceptionally(exception2); - return; } }); phaseTwoLoop(topic, reader, lh, outstanding, promise); @@ -443,35 +421,45 @@ private void phaseTwoLoop(String topic, Iterator> reader, CompletableFuture addToCompactedLedger( LedgerHandle lh, Message m, String topic, Semaphore outstanding) { + if (m == null) { + return flushBatchMessage(lh, topic, outstanding); + } + if (batchMessageContainer.haveEnoughSpace((MessageImpl) m)) { + batchMessageContainer.add((MessageImpl) m, null); + return CompletableFuture.completedFuture(false); + } + CompletableFuture f = flushBatchMessage(lh, topic, outstanding); + batchMessageContainer.add((MessageImpl) m, null); + return f; + } + + private CompletableFuture flushBatchMessage(LedgerHandle lh, String topic, + Semaphore outstanding) { + if (batchMessageContainer.getNumMessagesInBatch() <= 0) { + return CompletableFuture.completedFuture(false); + } CompletableFuture bkf = new CompletableFuture<>(); - if (m == null || batchMessageContainer.add((MessageImpl) m, null)) { - if (batchMessageContainer.getNumMessagesInBatch() > 0) { - try { - ByteBuf serialized = batchMessageContainer.toByteBuf(); - outstanding.acquire(); - mxBean.addCompactionWriteOp(topic, serialized.readableBytes()); - long start = System.nanoTime(); - lh.asyncAddEntry(serialized, - (rc, ledger, eid, ctx) -> { - outstanding.release(); - mxBean.addCompactionLatencyOp(topic, System.nanoTime() - start, TimeUnit.NANOSECONDS); - if (rc != BKException.Code.OK) { - bkf.completeExceptionally(BKException.create(rc)); - } else { - bkf.complete(true); - } - }, null); + try { + ByteBuf serialized = batchMessageContainer.toByteBuf(); + outstanding.acquire(); + mxBean.addCompactionWriteOp(topic, serialized.readableBytes()); + long start = System.nanoTime(); + lh.asyncAddEntry(serialized, + (rc, ledger, eid, ctx) -> { + outstanding.release(); + mxBean.addCompactionLatencyOp(topic, System.nanoTime() - start, TimeUnit.NANOSECONDS); + if (rc != BKException.Code.OK) { + bkf.completeExceptionally(BKException.create(rc)); + } else { + bkf.complete(true); + } + }, null); - } catch (Throwable t) { - log.error("Failed to add entry", t); - batchMessageContainer.discard((Exception) t); - return FutureUtil.failedFuture(t); - } - } else { - bkf.complete(false); - } - } else { - bkf.complete(false); + } catch (Throwable t) { + log.error("Failed to add entry", t); + batchMessageContainer.discard((Exception) t); + bkf.completeExceptionally(t); + return bkf; } return bkf; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java index 9b8b1e5efb99c..d79a31c07f218 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImplTest.java @@ -47,7 +47,6 @@ import org.apache.pulsar.compaction.CompactionTest; import org.testng.Assert; import org.testng.annotations.BeforeMethod; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class RawBatchMessageContainerImplTest { @@ -56,8 +55,6 @@ public class RawBatchMessageContainerImplTest { CryptoKeyReader cryptoKeyReader; Map encryptKeys; - int maxBytesInBatch = 5 * 1024 * 1024; - public void setEncryptionAndCompression(boolean encrypt, boolean compress) { if (compress) { compressionType = ZSTD; @@ -107,22 +104,22 @@ public MessageImpl createMessage(String topic, String value, int entryId) { public void setup() throws Exception { setEncryptionAndCompression(false, true); } - @DataProvider(name = "testBatchLimitByMessageCount") - public static Object[][] testBatchLimitByMessageCount() { - return new Object[][] {{true}, {false}}; - } - - @Test(timeOut = 20000, dataProvider = "testBatchLimitByMessageCount") - public void testToByteBufWithBatchLimit(boolean testBatchLimitByMessageCount) throws IOException { - RawBatchMessageContainerImpl container = testBatchLimitByMessageCount ? - new RawBatchMessageContainerImpl(2, Integer.MAX_VALUE) : - new RawBatchMessageContainerImpl(Integer.MAX_VALUE, 5); + @Test(timeOut = 20000) + public void testToByteBufWithBatchLimit()throws IOException { + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); String topic = "my-topic"; - var full1 = container.add(createMessage(topic, "hi-1", 0), null); - var full2 = container.add(createMessage(topic, "hi-2", 1), null); + MessageImpl message1 = createMessage(topic, "hi-1", 0); + boolean hasEnoughSpase1 = container.haveEnoughSpace(message1); + var full1 = container.add(message1, null); assertFalse(full1); - assertTrue(full2); + assertTrue(hasEnoughSpase1); + MessageImpl message2 = createMessage(topic, "hi-2", 1); + boolean hasEnoughSpase2 = container.haveEnoughSpace(message2); + assertFalse(hasEnoughSpase2); + var full2 = container.add(message2, null); + assertFalse(full2); + ByteBuf buf = container.toByteBuf(); @@ -167,7 +164,7 @@ public void testToByteBufWithBatchLimit(boolean testBatchLimitByMessageCount) th public void testToByteBufWithCompressionAndEncryption() throws IOException { setEncryptionAndCompression(true, true); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); container.setCryptoKeyReader(cryptoKeyReader); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); @@ -217,7 +214,7 @@ public void testToByteBufWithCompressionAndEncryption() throws IOException { @Test public void testToByteBufWithSingleMessage() throws IOException { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(2, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); ByteBuf buf = container.toByteBuf(); @@ -250,25 +247,31 @@ public void testToByteBufWithSingleMessage() throws IOException { } @Test - public void testMaxNumMessagesInBatch() { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); + public void testAddDifferentBatchMessage() { + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); String topic = "my-topic"; boolean isFull = container.add(createMessage(topic, "hi", 0), null); - Assert.assertTrue(isFull); - Assert.assertTrue(container.isBatchFull()); + Assert.assertFalse(isFull); + Assert.assertFalse(container.isBatchFull()); + MessageImpl message = createMessage(topic, "hi-1", 0); + Assert.assertTrue(container.haveEnoughSpace(message)); + isFull = container.add(message, null); + Assert.assertFalse(isFull); + message = createMessage(topic, "hi-2", 1); + Assert.assertFalse(container.haveEnoughSpace(message)); } @Test(expectedExceptions = UnsupportedOperationException.class) public void testCreateOpSendMsg() { - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); container.createOpSendMsg(); } @Test public void testToByteBufWithEncryptionWithoutCryptoKeyReader() { setEncryptionAndCompression(true, false); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); String topic = "my-topic"; container.add(createMessage(topic, "hi-1", 0), null); Assert.assertEquals(container.getNumMessagesInBatch(), 1); @@ -286,7 +289,7 @@ public void testToByteBufWithEncryptionWithoutCryptoKeyReader() { @Test public void testToByteBufWithEncryptionWithInvalidEncryptKeys() { setEncryptionAndCompression(true, false); - RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(1, maxBytesInBatch); + RawBatchMessageContainerImpl container = new RawBatchMessageContainerImpl(); container.setCryptoKeyReader(cryptoKeyReader); encryptKeys = new HashMap<>(); encryptKeys.put(null, null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index e603b3ccc4d12..f125022c1873e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -80,7 +80,6 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.ConsumerImpl; -import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; @@ -540,14 +539,8 @@ public void testBatchMessageIdsDontChange() throws Exception { Assert.assertEquals(message2.getKey(), "key2"); Assert.assertEquals(new String(message2.getData()), "my-message-3"); if (getCompactor() instanceof StrategicTwoPhaseCompactor) { - MessageIdImpl id = (MessageIdImpl) messages.get(0).getMessageId(); - MessageIdImpl id1 = new MessageIdImpl( - id.getLedgerId(), id.getEntryId(), id.getPartitionIndex()); - Assert.assertEquals(message1.getMessageId(), id1); - id = (MessageIdImpl) messages.get(2).getMessageId(); - MessageIdImpl id2 = new MessageIdImpl( - id.getLedgerId(), id.getEntryId(), id.getPartitionIndex()); - Assert.assertEquals(message2.getMessageId(), id2); + Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId()); + Assert.assertEquals(message2.getMessageId(), messages.get(1).getMessageId()); } else { Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId()); Assert.assertEquals(message2.getMessageId(), messages.get(2).getMessageId()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionRetentionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionRetentionTest.java index 1cac04c2fa956..e556ec8e0b200 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionRetentionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionRetentionTest.java @@ -34,7 +34,7 @@ public class StrategicCompactionRetentionTest extends CompactionRetentionTest { @Override public void setup() throws Exception { super.setup(); - compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler, 1); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); strategy = new TopicCompactionStrategyTest.DummyTopicCompactionStrategy(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java index 135a839bd54a8..54563431052eb 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java @@ -18,22 +18,33 @@ */ package org.apache.pulsar.compaction; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE; +import static org.testng.Assert.assertEquals; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.MessageIdAdv; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.TableView; import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats; import org.apache.pulsar.common.topics.TopicCompactionStrategy; +import org.apache.pulsar.common.util.FutureUtil; import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -47,7 +58,7 @@ public class StrategicCompactionTest extends CompactionTest { @Override public void setup() throws Exception { super.setup(); - compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler, 1); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); strategy = new TopicCompactionStrategyTest.DummyTopicCompactionStrategy(); } @@ -148,5 +159,58 @@ public void testNumericOrderCompaction() throws Exception { Assert.assertEquals(tableView.entrySet(), expectedCopy.entrySet()); } + @Test(timeOut = 20000) + public void testSameBatchCompactToSameBatch() throws Exception { + final String topic = + "persistent://my-property/use/my-ns/testSameBatchCompactToSameBatch" + UUID.randomUUID(); + + // Use odd number to make sure the last message is flush by `reader.hasNext() == false`. + final int messages = 11; + + // 1.create producer and publish message to the topic. + ProducerBuilder builder = pulsarClient.newProducer(Schema.INT32) + .compressionType(MSG_COMPRESSION_TYPE).topic(topic); + builder.batchingMaxMessages(2) + .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS); + + Producer producer = builder.create(); + + List> futures = new ArrayList<>(messages); + for (int i = 0; i < messages; i++) { + futures.add(producer.newMessage().key(String.valueOf(i)) + .value(i) + .sendAsync()); + } + FutureUtil.waitForAll(futures).get(); + + // 2.compact the topic. + StrategicTwoPhaseCompactor compactor + = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + compactor.compact(topic, strategy).get(); + // consumer with readCompacted enabled only get compacted entries + try (Consumer consumer = pulsarClient + .newConsumer(Schema.INT32) + .topic(topic) + .subscriptionName("sub1") + .readCompacted(true) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) { + int received = 0; + while (true) { + Message m = consumer.receive(2, TimeUnit.SECONDS); + if (m == null) { + break; + } + MessageIdAdv messageId = (MessageIdAdv) m.getMessageId(); + if (received < messages - 1) { + assertEquals(messageId.getBatchSize(), 2); + } else { + assertEquals(messageId.getBatchSize(), 0); + } + received++; + } + assertEquals(received, messages); + } + + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactorTest.java index 91dd8a2bd358b..bc65791b323cd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactorTest.java @@ -33,7 +33,7 @@ public class StrategicCompactorTest extends CompactorTest { @Override public void setup() throws Exception { super.setup(); - compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler, 1); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); strategy = new TopicCompactionStrategyTest.DummyTopicCompactionStrategy(); } @@ -46,4 +46,4 @@ protected long compact(String topic) throws ExecutionException, InterruptedExcep protected Compactor getCompactor() { return compactor; } -} \ No newline at end of file +} From 8b1c2a4d3a96cc8c637dbce73ce9a9c2adef7edf Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Fri, 15 Sep 2023 20:17:55 +0800 Subject: [PATCH 284/494] [fix][bk] Improve to the ReplicaitonWorker performance by deleting invalid underreplication nodes (#21160) --- .../PulsarLedgerUnderreplicationManager.java | 29 ++++++++ .../LedgerUnderreplicationManagerTest.java | 66 +++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java index 79fdc44cb2b06..dda8d7256ed52 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java @@ -25,10 +25,12 @@ import static org.apache.bookkeeper.proto.DataFormats.PlacementPolicyCheckFormat; import static org.apache.bookkeeper.proto.DataFormats.ReplicasCheckFormat; import static org.apache.bookkeeper.proto.DataFormats.UnderreplicatedLedgerFormat; +import com.google.common.base.Joiner; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -61,6 +63,8 @@ import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.CreateOption; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.zookeeper.KeeperException; @Slf4j public class PulsarLedgerUnderreplicationManager implements LedgerUnderreplicationManager { @@ -393,6 +397,31 @@ public void markLedgerReplicated(long ledgerId) throws ReplicationException.Unav Lock l = heldLocks.get(ledgerId); if (l != null) { store.delete(getUrLedgerPath(ledgerId), Optional.of(l.getLedgerNodeVersion())).get(); + if (store instanceof ZKMetadataStore) { + try { + // clean up the hierarchy + String[] parts = getUrLedgerPath(ledgerId).split("/"); + for (int i = 1; i <= 4; i++) { + String[] p = Arrays.copyOf(parts, parts.length - i); + String path = Joiner.on("/").join(p); + Optional getResult = store.get(path).get(); + if (getResult.isPresent()) { + store.delete(path, Optional.of(getResult.get().getStat().getVersion())).get(); + } + } + } catch (ExecutionException ee) { + // This can happen when cleaning up the hierarchy. + // It's safe to ignore, it simply means another + // ledger in the same hierarchy has been marked as + // underreplicated. + if (ee.getCause() instanceof MetadataStoreException && ee.getCause().getCause() + instanceof KeeperException.NotEmptyException) { + //do nothing. + } else { + log.warn("Error deleting underrepcalited ledger parent node", ee); + } + } + } } } catch (ExecutionException ee) { if (ee.getCause() instanceof MetadataStoreException.NotFoundException) { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java index 0df325b3c57a0..649dc1663c68f 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java @@ -23,12 +23,14 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.protobuf.TextFormat; +import java.lang.reflect.Field; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -54,6 +56,7 @@ import org.apache.bookkeeper.util.BookKeeperConstants; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.metadata.BaseMetadataStoreTest; +import org.apache.pulsar.metadata.api.GetResult; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -296,6 +299,69 @@ public void testMarkingAsReplicated(String provider, Supplier urlSupplie assertEquals(l, lB.get(), "Should be the ledger I marked"); } + + @Test(timeOut = 10000) + public void testZkMetasStoreMarkReplicatedDeleteEmptyParentNodes() throws Exception { + methodSetup(stringSupplier(() -> zks.getConnectionString())); + + String missingReplica = "localhost:3181"; + + @Cleanup + LedgerUnderreplicationManager m1 = lmf.newLedgerUnderreplicationManager(); + + Long ledgerA = 0xfeadeefdacL; + m1.markLedgerUnderreplicated(ledgerA, missingReplica); + + Field storeField = m1.getClass().getDeclaredField("store"); + storeField.setAccessible(true); + MetadataStoreExtended metadataStore = (MetadataStoreExtended) storeField.get(m1); + + String fiveLevelPath = PulsarLedgerUnderreplicationManager.getUrLedgerPath(urLedgerPath, ledgerA); + Optional getResult = metadataStore.get(fiveLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + String fourLevelPath = fiveLevelPath.substring(0, fiveLevelPath.lastIndexOf("/")); + getResult = metadataStore.get(fourLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + String threeLevelPath = fourLevelPath.substring(0, fourLevelPath.lastIndexOf("/")); + getResult = metadataStore.get(threeLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + String twoLevelPath = fourLevelPath.substring(0, threeLevelPath.lastIndexOf("/")); + getResult = metadataStore.get(twoLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + String oneLevelPath = fourLevelPath.substring(0, twoLevelPath.lastIndexOf("/")); + getResult = metadataStore.get(oneLevelPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + getResult = metadataStore.get(urLedgerPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + + long ledgerToRereplicate = m1.getLedgerToRereplicate(); + assertEquals(ledgerToRereplicate, ledgerA); + m1.markLedgerReplicated(ledgerA); + + getResult = metadataStore.get(fiveLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(fourLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(threeLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(twoLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(oneLevelPath).get(1, TimeUnit.SECONDS); + assertFalse(getResult.isPresent()); + + getResult = metadataStore.get(urLedgerPath).get(1, TimeUnit.SECONDS); + assertTrue(getResult.isPresent()); + } + /** * Test releasing of a ledger * A ledger is released when a client decides it does not want From 1c851db1b113ed807223f6430cb8ec6b815261fc Mon Sep 17 00:00:00 2001 From: AloysZhang Date: Mon, 18 Sep 2023 17:12:58 +0800 Subject: [PATCH 285/494] [fix][broker] Backport fix UniformLoadShedder selecet wrong overloadbroker and underloadbroker (#21179) --- .../loadbalance/impl/UniformLoadShedder.java | 163 +++++++++++------- .../impl/UniformLoadShedderTest.java | 78 +++++++++ 2 files changed, 178 insertions(+), 63 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedder.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedder.java index b92af5b7c69f3..d8dcfa007cfc5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedder.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedder.java @@ -25,7 +25,7 @@ import org.apache.commons.lang3.mutable.MutableDouble; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.mutable.MutableObject; -import org.apache.commons.lang3.tuple.Triple; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.loadbalance.LoadData; import org.apache.pulsar.broker.loadbalance.LoadSheddingStrategy; @@ -36,7 +36,7 @@ /** * This strategy tends to distribute load uniformly across all brokers. This strategy checks load difference between - * broker with highest load and broker with lowest load. If the difference is higher than configured thresholds + * broker with the highest load and broker with the lowest load. If the difference is higher than configured thresholds * {@link ServiceConfiguration#getLoadBalancerMsgRateDifferenceShedderThreshold()} or * {@link ServiceConfiguration#getLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold()} then it finds out * bundles which can be unloaded to distribute traffic evenly across all brokers. @@ -63,25 +63,37 @@ public Multimap findBundlesForUnloading(final LoadData loadData, Map loadBundleData = loadData.getBundleDataForLoadShedding(); Map recentlyUnloadedBundles = loadData.getRecentlyUnloadedBundles(); - MutableObject overloadedBroker = new MutableObject<>(); - MutableObject underloadedBroker = new MutableObject<>(); + MutableObject msgRateOverloadedBroker = new MutableObject<>(); + MutableObject msgThroughputOverloadedBroker = new MutableObject<>(); + MutableObject msgRateUnderloadedBroker = new MutableObject<>(); + MutableObject msgThroughputUnderloadedBroker = new MutableObject<>(); MutableDouble maxMsgRate = new MutableDouble(-1); - MutableDouble maxThroughputRate = new MutableDouble(-1); + MutableDouble maxThroughput = new MutableDouble(-1); MutableDouble minMsgRate = new MutableDouble(Integer.MAX_VALUE); - MutableDouble minThroughputRate = new MutableDouble(Integer.MAX_VALUE); + MutableDouble minThroughput = new MutableDouble(Integer.MAX_VALUE); + brokersData.forEach((broker, data) -> { double msgRate = data.getLocalData().getMsgRateIn() + data.getLocalData().getMsgRateOut(); double throughputRate = data.getLocalData().getMsgThroughputIn() + data.getLocalData().getMsgThroughputOut(); - if (msgRate > maxMsgRate.getValue() || throughputRate > maxThroughputRate.getValue()) { - overloadedBroker.setValue(broker); + if (msgRate > maxMsgRate.getValue()) { + msgRateOverloadedBroker.setValue(broker); maxMsgRate.setValue(msgRate); - maxThroughputRate.setValue(throughputRate); } - if (msgRate < minMsgRate.getValue() || throughputRate < minThroughputRate.getValue()) { - underloadedBroker.setValue(broker); + + if (throughputRate > maxThroughput.getValue()) { + msgThroughputOverloadedBroker.setValue(broker); + maxThroughput.setValue(throughputRate); + } + + if (msgRate < minMsgRate.getValue()) { + msgRateUnderloadedBroker.setValue(broker); minMsgRate.setValue(msgRate); - minThroughputRate.setValue(throughputRate); + } + + if (throughputRate < minThroughput.getValue()) { + msgThroughputUnderloadedBroker.setValue(broker); + minThroughput.setValue(throughputRate); } }); @@ -91,12 +103,12 @@ public Multimap findBundlesForUnloading(final LoadData loadData, if (minMsgRate.getValue() <= EPS && minMsgRate.getValue() >= -EPS) { minMsgRate.setValue(1.0); } - if (minThroughputRate.getValue() <= EPS && minThroughputRate.getValue() >= -EPS) { - minThroughputRate.setValue(1.0); + if (minThroughput.getValue() <= EPS && minThroughput.getValue() >= -EPS) { + minThroughput.setValue(1.0); } double msgRateDifferencePercentage = ((maxMsgRate.getValue() - minMsgRate.getValue()) * 100) / (minMsgRate.getValue()); - double msgThroughputDifferenceRate = maxThroughputRate.getValue() / minThroughputRate.getValue(); + double msgThroughputDifferenceRate = maxThroughput.getValue() / minThroughput.getValue(); // if the threshold matches then find out how much load needs to be unloaded by considering number of msgRate // and throughput. @@ -105,66 +117,91 @@ public Multimap findBundlesForUnloading(final LoadData loadData, boolean isMsgThroughputThresholdExceeded = conf .getLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold() > 0 && msgThroughputDifferenceRate > conf - .getLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(); + .getLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(); if (isMsgRateThresholdExceeded || isMsgThroughputThresholdExceeded) { - if (log.isDebugEnabled()) { - log.debug( - "Found bundles for uniform load balancing. " - + "overloaded broker {} with (msgRate,throughput)= ({},{}) " - + "and underloaded broker {} with (msgRate,throughput)= ({},{})", - overloadedBroker.getValue(), maxMsgRate.getValue(), maxThroughputRate.getValue(), - underloadedBroker.getValue(), minMsgRate.getValue(), minThroughputRate.getValue()); - } MutableInt msgRateRequiredFromUnloadedBundles = new MutableInt( (int) ((maxMsgRate.getValue() - minMsgRate.getValue()) * conf.getMaxUnloadPercentage())); MutableInt msgThroughputRequiredFromUnloadedBundles = new MutableInt( - (int) ((maxThroughputRate.getValue() - minThroughputRate.getValue()) + (int) ((maxThroughput.getValue() - minThroughput.getValue()) * conf.getMaxUnloadPercentage())); - LocalBrokerData overloadedBrokerData = brokersData.get(overloadedBroker.getValue()).getLocalData(); - - if (overloadedBrokerData.getBundles().size() > 1 - && (msgRateRequiredFromUnloadedBundles.getValue() >= conf.getMinUnloadMessage() - || msgThroughputRequiredFromUnloadedBundles.getValue() >= conf.getMinUnloadMessageThroughput())) { - // Sort bundles by throughput, then pick the bundle which can help to reduce load uniformly with - // under-loaded broker - loadBundleData.entrySet().stream() - .filter(e -> overloadedBrokerData.getBundles().contains(e.getKey())) - .map((e) -> { - String bundle = e.getKey(); - BundleData bundleData = e.getValue(); - TimeAverageMessageData shortTermData = bundleData.getShortTermData(); - double throughput = isMsgRateThresholdExceeded - ? shortTermData.getMsgRateIn() + shortTermData.getMsgRateOut() - : shortTermData.getMsgThroughputIn() + shortTermData.getMsgThroughputOut(); - return Triple.of(bundle, bundleData, throughput); - }).filter(e -> !recentlyUnloadedBundles.containsKey(e.getLeft())) - .sorted((e1, e2) -> Double.compare(e2.getRight(), e1.getRight())).forEach((e) -> { - if (conf.getMaxUnloadBundleNumPerShedding() != -1 - && selectedBundlesCache.size() >= conf.getMaxUnloadBundleNumPerShedding()) { - return; - } - String bundle = e.getLeft(); - BundleData bundleData = e.getMiddle(); - TimeAverageMessageData shortTermData = bundleData.getShortTermData(); - double throughput = shortTermData.getMsgThroughputIn() - + shortTermData.getMsgThroughputOut(); - double bundleMsgRate = shortTermData.getMsgRateIn() + shortTermData.getMsgRateOut(); - if (isMsgRateThresholdExceeded) { + if (isMsgRateThresholdExceeded) { + if (log.isDebugEnabled()) { + log.debug("Found bundles for uniform load balancing. " + + "msgRate overloaded broker: {} with msgRate: {}, " + + "msgRate underloaded broker: {} with msgRate: {}", + msgRateOverloadedBroker.getValue(), maxMsgRate.getValue(), + msgRateUnderloadedBroker.getValue(), minMsgRate.getValue()); + } + LocalBrokerData overloadedBrokerData = + brokersData.get(msgRateOverloadedBroker.getValue()).getLocalData(); + if (overloadedBrokerData.getBundles().size() > 1 + && (msgRateRequiredFromUnloadedBundles.getValue() >= conf.getMinUnloadMessage())) { + // Sort bundles by throughput, then pick the bundle which can help to reduce load uniformly with + // under-loaded broker + loadBundleData.entrySet().stream() + .filter(e -> overloadedBrokerData.getBundles().contains(e.getKey())) + .map((e) -> { + String bundle = e.getKey(); + TimeAverageMessageData shortTermData = e.getValue().getShortTermData(); + double msgRate = shortTermData.getMsgRateIn() + shortTermData.getMsgRateOut(); + return Pair.of(bundle, msgRate); + }).filter(e -> !recentlyUnloadedBundles.containsKey(e.getLeft())) + .sorted((e1, e2) -> Double.compare(e2.getRight(), e1.getRight())).forEach((e) -> { + if (conf.getMaxUnloadBundleNumPerShedding() != -1 + && selectedBundlesCache.size() >= conf.getMaxUnloadBundleNumPerShedding()) { + return; + } + String bundle = e.getLeft(); + double bundleMsgRate = e.getRight(); if (bundleMsgRate <= (msgRateRequiredFromUnloadedBundles.getValue() + 1000/* delta */)) { log.info("Found bundle to unload with msgRate {}", bundleMsgRate); msgRateRequiredFromUnloadedBundles.add(-bundleMsgRate); - selectedBundlesCache.put(overloadedBroker.getValue(), bundle); + selectedBundlesCache.put(msgRateOverloadedBroker.getValue(), bundle); } - } else { - if (throughput <= (msgThroughputRequiredFromUnloadedBundles.getValue())) { - log.info("Found bundle to unload with throughput {}", throughput); - msgThroughputRequiredFromUnloadedBundles.add(-throughput); - selectedBundlesCache.put(overloadedBroker.getValue(), bundle); + }); + } + } else { + if (log.isDebugEnabled()) { + log.debug("Found bundles for uniform load balancing. " + + "msgThroughput overloaded broker: {} with msgThroughput {}, " + + "msgThroughput underloaded broker: {} with msgThroughput: {}", + msgThroughputOverloadedBroker.getValue(), maxThroughput.getValue(), + msgThroughputUnderloadedBroker.getValue(), minThroughput.getValue()); + } + LocalBrokerData overloadedBrokerData = + brokersData.get(msgThroughputOverloadedBroker.getValue()).getLocalData(); + if (overloadedBrokerData.getBundles().size() > 1 + && + msgThroughputRequiredFromUnloadedBundles.getValue() >= conf.getMinUnloadMessageThroughput()) { + // Sort bundles by throughput, then pick the bundle which can help to reduce load uniformly with + // under-loaded broker + loadBundleData.entrySet().stream() + .filter(e -> overloadedBrokerData.getBundles().contains(e.getKey())) + .map((e) -> { + String bundle = e.getKey(); + TimeAverageMessageData shortTermData = e.getValue().getShortTermData(); + double msgThroughput = shortTermData.getMsgThroughputIn() + + shortTermData.getMsgThroughputOut(); + return Pair.of(bundle, msgThroughput); + }).filter(e -> !recentlyUnloadedBundles.containsKey(e.getLeft())) + .sorted((e1, e2) -> Double.compare(e2.getRight(), e1.getRight())).forEach((e) -> { + if (conf.getMaxUnloadBundleNumPerShedding() != -1 + && selectedBundlesCache.size() >= conf.getMaxUnloadBundleNumPerShedding()) { + return; } - } - }); + String bundle = e.getLeft(); + double msgThroughput = e.getRight(); + if (msgThroughput <= (msgThroughputRequiredFromUnloadedBundles.getValue() + + 1000/* delta */)) { + log.info("Found bundle to unload with msgThroughput {}", msgThroughput); + msgThroughputRequiredFromUnloadedBundles.add(-msgThroughput); + selectedBundlesCache.put(msgThroughputOverloadedBroker.getValue(), bundle); + } + }); + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedderTest.java index 00182fffb8a31..4b4042cf31a72 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedderTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/UniformLoadShedderTest.java @@ -26,6 +26,7 @@ import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; @Test(groups = "broker") public class UniformLoadShedderTest { @@ -119,4 +120,81 @@ public void testBrokerWithMultipleBundles() { assertFalse(bundlesToUnload.isEmpty()); } + @Test + public void testOverloadBrokerSelect() { + conf.setMaxUnloadBundleNumPerShedding(1); + conf.setMaxUnloadPercentage(0.5); + int numBrokers = 5; + int numBundles = 5; + LoadData loadData = new LoadData(); + + LocalBrokerData[] localBrokerDatas = new LocalBrokerData[]{ + new LocalBrokerData(), + new LocalBrokerData(), + new LocalBrokerData(), + new LocalBrokerData(), + new LocalBrokerData()}; + + String[] brokerNames = new String[]{"broker0", "broker1", "broker2", "broker3", "broker4"}; + + double[] brokerMsgRates = new double[]{ + 50000, // broker0 + 60000, // broker1 + 70000, // broker2 + 10000, // broker3 + 20000};// broker4 + + double[] brokerMsgThroughputs = new double[]{ + 50 * 1024 * 1024, // broker0 + 60 * 1024 * 1024, // broker1 + 70 * 1024 * 1024, // broker2 + 80 * 1024 * 1024, // broker3 + 10 * 1024 * 1024};// broker4 + + + for (int brokerId = 0; brokerId < numBrokers; brokerId++) { + double msgRate = brokerMsgRates[brokerId] / numBundles; + double throughput = brokerMsgThroughputs[brokerId] / numBundles; + for (int i = 0; i < numBundles; ++i) { + String bundleName = "broker-" + brokerId + "-bundle-" + i; + localBrokerDatas[brokerId].getBundles().add(bundleName); + localBrokerDatas[brokerId].setMsgRateIn(brokerMsgRates[brokerId]); + localBrokerDatas[brokerId].setMsgThroughputIn(brokerMsgThroughputs[brokerId]); + BundleData bundle = new BundleData(); + + TimeAverageMessageData timeAverageMessageData = new TimeAverageMessageData(); + timeAverageMessageData.setMsgRateIn(msgRate); + timeAverageMessageData.setMsgThroughputIn(throughput); + bundle.setShortTermData(timeAverageMessageData); + loadData.getBundleData().put(bundleName, bundle); + } + loadData.getBrokerData().put(brokerNames[brokerId], new BrokerData(localBrokerDatas[brokerId])); + } + + // disable throughput based load shedding, enable rate based load shedding only + conf.setLoadBalancerMsgRateDifferenceShedderThreshold(50); + conf.setLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(0); + + Multimap bundlesToUnload = uniformLoadShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 1); + assertTrue(bundlesToUnload.containsKey("broker2")); + + + // disable rate based load shedding, enable throughput based load shedding only + conf.setLoadBalancerMsgRateDifferenceShedderThreshold(0); + conf.setLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(2); + + bundlesToUnload = uniformLoadShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 1); + assertTrue(bundlesToUnload.containsKey("broker3")); + + // enable both rate and throughput based load shedding, but rate based load shedding has higher priority + conf.setLoadBalancerMsgRateDifferenceShedderThreshold(50); + conf.setLoadBalancerMsgThroughputMultiplierDifferenceShedderThreshold(2); + + bundlesToUnload = uniformLoadShedder.findBundlesForUnloading(loadData, conf); + assertEquals(bundlesToUnload.size(), 1); + assertTrue(bundlesToUnload.containsKey("broker2")); + } + } From 3a30526ac936101d00eec6c053a6119c091be382 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 18 Sep 2023 18:15:14 +0800 Subject: [PATCH 286/494] [fix][broker] Fix write duplicate entries into the compacted ledger after RawReader reconnects (#21081) (#21165) --- .../pulsar/compaction/TwoPhaseCompactor.java | 13 +++- .../pulsar/compaction/CompactionTest.java | 66 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java index 886a523df0257..be08bd81c1e49 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java @@ -207,7 +207,7 @@ private CompletableFuture phaseTwoSeekThenLoop(RawReader reader, MessageId reader.seekAsync(from).thenCompose((v) -> { Semaphore outstanding = new Semaphore(MAX_OUTSTANDING); CompletableFuture loopPromise = new CompletableFuture<>(); - phaseTwoLoop(reader, to, latestForKey, ledger, outstanding, loopPromise); + phaseTwoLoop(reader, to, latestForKey, ledger, outstanding, loopPromise, MessageId.earliest); return loopPromise; }).thenCompose((v) -> closeLedger(ledger)) .thenCompose((v) -> reader.acknowledgeCumulativeAsync(lastReadId, @@ -229,7 +229,8 @@ private CompletableFuture phaseTwoSeekThenLoop(RawReader reader, MessageId } private void phaseTwoLoop(RawReader reader, MessageId to, Map latestForKey, - LedgerHandle lh, Semaphore outstanding, CompletableFuture promise) { + LedgerHandle lh, Semaphore outstanding, CompletableFuture promise, + MessageId lastCompactedMessageId) { if (promise.isDone()) { return; } @@ -238,6 +239,12 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map m.close(); return; } + + if (m.getMessageId().compareTo(lastCompactedMessageId) <= 0) { + phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise, lastCompactedMessageId); + return; + } + try { MessageId id = m.getMessageId(); Optional messageToAdd = Optional.empty(); @@ -308,7 +315,7 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map } return; } - phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise); + phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise, m.getMessageId()); } finally { m.close(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index f125022c1873e..535d6715a3055 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -1870,4 +1870,70 @@ public void testReceiverQueueSize() throws Exception { consumer.close(); producer.close(); } + + @Test + public void testCompactionDuplicate() throws Exception { + String topic = "persistent://my-property/use/my-ns/testCompactionDuplicate"; + final int numMessages = 1000; + final int maxKeys = 800; + + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + // trigger compaction (create __compaction cursor) + admin.topics().triggerCompaction(topic); + + Map expected = new HashMap<>(); + Random r = new Random(0); + + pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close(); + + for (int j = 0; j < numMessages; j++) { + int keyIndex = r.nextInt(maxKeys); + String key = "key" + keyIndex; + byte[] data = ("my-message-" + key + "-" + j).getBytes(); + producer.newMessage().key(key).value(data).send(); + expected.put(key, data); + } + + producer.flush(); + + // trigger compaction + admin.topics().triggerCompaction(topic); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topic).status, + LongRunningProcessStatus.Status.RUNNING); + }); + + // Wait for phase one to complete + Thread.sleep(500); + + // Unload topic make reader of compaction reconnect + admin.topics().unload(topic); + + Awaitility.await().untilAsserted(() -> { + PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(topic, false); + // Compacted topic ledger should have same number of entry equals to number of unique key. + Assert.assertEquals(internalStats.compactedLedger.entries, expected.size()); + Assert.assertTrue(internalStats.compactedLedger.ledgerId > -1); + Assert.assertFalse(internalStats.compactedLedger.offloaded); + }); + + // consumer with readCompacted enabled only get compacted entries + try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1") + .readCompacted(true).subscribe()) { + while (true) { + Message m = consumer.receive(2, TimeUnit.SECONDS); + Assert.assertEquals(expected.remove(m.getKey()), m.getData()); + if (expected.isEmpty()) { + break; + } + } + } + } } From 7e0c0f8341dc39513a2914d634edb4824ca5dec3 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 22 Sep 2023 10:57:44 +0800 Subject: [PATCH 287/494] [improve][broker][branch-3.0] Make read compacted entries support maxReadSizeBytes limitation (#21065) (#21164) --- .../mledger/impl/ManagedCursorImpl.java | 4 +- ...sistentDispatcherSingleActiveConsumer.java | 2 +- ...reamingDispatcherSingleActiveConsumer.java | 2 +- .../pulsar/compaction/CompactedTopic.java | 3 +- .../pulsar/compaction/CompactedTopicImpl.java | 15 ++++- .../pulsar/compaction/CompactionTest.java | 56 +++++++++++++++++++ .../compaction/StrategicCompactionTest.java | 22 +++++++- 7 files changed, 96 insertions(+), 8 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 3631feccca946..4500d27785240 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -3506,7 +3506,7 @@ public ManagedCursorMXBean getStats() { return this.mbean; } - void updateReadStats(int readEntriesCount, long readEntriesSize) { + public void updateReadStats(int readEntriesCount, long readEntriesSize) { this.entriesReadCount += readEntriesCount; this.entriesReadSize += readEntriesSize; } @@ -3538,7 +3538,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { }, null); } - private int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { + public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) { if (maxSizeBytes == NO_MAX_SIZE_LIMIT) { return maxEntries; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java index 7cbf7bd2c787a..67265f626a19b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentDispatcherSingleActiveConsumer.java @@ -347,7 +347,7 @@ protected void readMoreEntries(Consumer consumer) { } havePendingRead = true; if (consumer.readCompacted()) { - topic.getCompactedTopic().asyncReadEntriesOrWait(cursor, messagesToRead, isFirstRead, + topic.getCompactedTopic().asyncReadEntriesOrWait(cursor, messagesToRead, bytesToRead, isFirstRead, this, consumer); } else { ReadEntriesCtx readEntriesCtx = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java index efe9de778a3e7..22dcc48994c1d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentStreamingDispatcherSingleActiveConsumer.java @@ -217,7 +217,7 @@ protected void readMoreEntries(Consumer consumer) { havePendingRead = true; if (consumer.readCompacted()) { - topic.getCompactedTopic().asyncReadEntriesOrWait(cursor, messagesToRead, isFirstRead, + topic.getCompactedTopic().asyncReadEntriesOrWait(cursor, messagesToRead, bytesToRead, isFirstRead, this, consumer); } else { streamingEntryReader.asyncReadEntries(messagesToRead, bytesToRead, consumer); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java index e1a10b3bbb212..c575a872fd60a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopic.java @@ -30,7 +30,8 @@ public interface CompactedTopic { CompletableFuture newCompactedLedger(Position p, long compactedLedgerId); CompletableFuture deleteCompactedLedger(long compactedLedgerId); void asyncReadEntriesOrWait(ManagedCursor cursor, - int numberOfEntriesToRead, + int maxEntries, + long bytesToRead, boolean isFirstRead, ReadEntriesCallback callback, Consumer consumer); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index a0c45aa750f7a..703ba688d3bdc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -42,6 +42,7 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.impl.EntryImpl; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.service.Consumer; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer.ReadEntriesCtx; @@ -87,7 +88,8 @@ public CompletableFuture deleteCompactedLedger(long compactedLedgerId) { @Override public void asyncReadEntriesOrWait(ManagedCursor cursor, - int numberOfEntriesToRead, + int maxEntries, + long bytesToRead, boolean isFirstRead, ReadEntriesCallback callback, Consumer consumer) { synchronized (this) { @@ -102,8 +104,11 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, ReadEntriesCtx readEntriesCtx = ReadEntriesCtx.create(consumer, DEFAULT_CONSUMER_EPOCH); if (compactionHorizon == null || compactionHorizon.compareTo(cursorPosition) < 0) { - cursor.asyncReadEntriesOrWait(numberOfEntriesToRead, callback, readEntriesCtx, PositionImpl.LATEST); + cursor.asyncReadEntriesOrWait(maxEntries, bytesToRead, callback, readEntriesCtx, PositionImpl.LATEST); } else { + ManagedCursorImpl managedCursor = (ManagedCursorImpl) cursor; + int numberOfEntriesToRead = managedCursor.applyMaxSizeCap(maxEntries, bytesToRead); + compactedTopicContext.thenCompose( (context) -> findStartPoint(cursorPosition, context.ledger.getLastAddConfirmed(), context.cache) .thenCompose((startPoint) -> { @@ -128,6 +133,12 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, } return readEntries(context.ledger, startPoint, endPoint) .thenAccept((entries) -> { + long entriesSize = 0; + for (Entry entry : entries) { + entriesSize += entry.getLength(); + } + managedCursor.updateReadStats(entries.size(), entriesSize); + Entry lastEntry = entries.get(entries.size() - 1); // The compaction task depends on the last snapshot and the incremental // entries to build the new snapshot. So for the compaction cursor, we diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index 535d6715a3055..3985069c6eba8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -25,6 +25,7 @@ import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertSame; import static org.testng.Assert.assertTrue; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -55,11 +56,14 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.bookkeeper.mledger.Position; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer; +import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.SystemTopic; import org.apache.pulsar.client.admin.LongRunningProcessStatus; @@ -89,6 +93,7 @@ import org.apache.pulsar.common.protocol.Markers; import org.apache.pulsar.common.util.FutureUtil; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -1871,6 +1876,57 @@ public void testReceiverQueueSize() throws Exception { producer.close(); } + @Test + public void testDispatcherMaxReadSizeBytes() throws Exception { + final String topicName = + "persistent://my-property/use/my-ns/testDispatcherMaxReadSizeBytes" + UUID.randomUUID(); + final String subName = "my-sub"; + final int receiveQueueSize = 1; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + Producer producer = pulsarClient.newProducer(Schema.BYTES) + .topic(topicName).create(); + + for (int i = 0; i < 10; i+=2) { + producer.newMessage().key(null).value(new byte[4*1024*1024]).send(); + } + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + admin.topics().unload(topicName); + + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.BYTES) + .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) + .subscribe(); + + PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get(); + PersistentSubscription persistentSubscription = topic.getSubscriptions().get(subName); + PersistentDispatcherSingleActiveConsumer dispatcher = + Mockito.spy((PersistentDispatcherSingleActiveConsumer) persistentSubscription.getDispatcher()); + FieldUtils.writeDeclaredField(persistentSubscription, "dispatcher", dispatcher, true); + + Awaitility.await().untilAsserted(() -> { + assertSame(consumer.getStats().getMsgNumInReceiverQueue(), 1); + }); + + consumer.increaseAvailablePermits(2); + + Thread.sleep(2000); + + Mockito.verify(dispatcher, Mockito.atLeastOnce()) + .readEntriesComplete(Mockito.argThat(argument -> argument.size() == 1), + Mockito.any(PersistentDispatcherSingleActiveConsumer.ReadEntriesCtx.class)); + + consumer.close(); + producer.close(); + } + @Test public void testCompactionDuplicate() throws Exception { String topic = "persistent://my-property/use/my-ns/testCompactionDuplicate"; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java index 54563431052eb..9e327ef90fae4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/StrategicCompactionTest.java @@ -20,7 +20,6 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MSG_COMPRESSION_TYPE; import static org.testng.Assert.assertEquals; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -213,4 +212,25 @@ public void testSameBatchCompactToSameBatch() throws Exception { } } + + @Override + public void testCompactCompressedBatching() throws Exception { + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + super.testCompactCompressedBatching(); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + } + + @Override + public void testCompactEncryptedAndCompressedBatching() throws Exception { + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + super.testCompactEncryptedAndCompressedBatching(); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + } + + @Override + public void testCompactEncryptedBatching() throws Exception { + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + super.testCompactEncryptedBatching(); + compactor = new StrategicTwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + } } From 9a7c4bbce69502d91f6ca5249a617471decd501f Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 21 Sep 2023 23:18:45 +0800 Subject: [PATCH 288/494] [improve] [proxy] Not close the socket if lookup failed caused by too many requests (#21216) Motivation: The Pulsar client will close the socket if it receives a `ServiceNotReady` error when doing a lookup. The Broker will respond to the client with a `TooManyRequests` error if there are too many lookup requests in progress, but the Pulsar Proxy responds to the client with a `ServiceNotReady` error in the same scenario. Modifications: Make Pulsar Proxy respond to the client with a `TooManyRequests` error if there are too many lookup requests in progress. (cherry picked from commit d6c3fa42059d96b04b4132ccc9256c3e76d26959) --- .../proxy/server/LookupProxyHandler.java | 2 +- .../server/ProxyLookupThrottlingTest.java | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java index b62b3bacf0114..f76adadcc3e0a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/LookupProxyHandler.java @@ -116,7 +116,7 @@ public void handleLookup(CommandLookupTopic lookup) { log.debug("Lookup Request ID {} from {} rejected - {}.", clientRequestId, clientAddress, throttlingErrorMessage); } - writeAndFlush(Commands.newLookupErrorResponse(ServerError.ServiceNotReady, + writeAndFlush(Commands.newLookupErrorResponse(ServerError.TooManyRequests, throttlingErrorMessage, clientRequestId)); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java index 4861117ef6ff5..1b63aa14dfe42 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyLookupThrottlingTest.java @@ -20,18 +20,26 @@ import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import java.util.Optional; +import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import lombok.Cleanup; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.BinaryProtoLookupService; +import org.apache.pulsar.client.impl.ClientCnx; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.mockito.Mockito; import org.testng.Assert; @@ -112,4 +120,32 @@ public void testLookup() throws Exception { Assert.assertEquals(LookupProxyHandler.REJECTED_PARTITIONS_METADATA_REQUESTS.get(), 5.0d); } + + @Test + public void testLookupThrottling() throws Exception { + PulsarClientImpl client = (PulsarClientImpl) PulsarClient.builder() + .serviceUrl(proxyService.getServiceUrl()).build(); + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + LookupService lookupService = client.getLookup(); + assertTrue(lookupService instanceof BinaryProtoLookupService); + ClientCnx lookupConnection = client.getCnxPool().getConnection(lookupService.resolveHost()).join(); + + // Make no permits to lookup. + Semaphore lookupSemaphore = proxyService.getLookupRequestSemaphore(); + int availablePermits = lookupSemaphore.availablePermits(); + lookupSemaphore.acquire(availablePermits); + + // Verify will receive too many request exception, and the socket will not be closed. + try { + lookupService.getBroker(TopicName.get(tpName)).get(); + fail("Expected too many request error."); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Too many")); + } + assertTrue(lookupConnection.ctx().channel().isActive()); + + // cleanup. + lookupSemaphore.release(availablePermits); + client.close(); + } } From 66054d5d4c2604da031674d3df2a4a1954fae538 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 22 Sep 2023 16:26:04 +0800 Subject: [PATCH 289/494] [improve] [client] Merge lookup requests for the same topic (#21232) Motivation: Multiple consumers and producers can be maintained by the same Pulsar Client. In some cases, multiple consumers or producers might attempt to connect to the same topic. To optimize the process, it is recommended to perform the topic lookup only once for each topic. Modifications: - Merge lookup requests for the same topic. - Merge get partitioned metadata request for the same partitioned topic. (cherry picked from commit be4ab66bb8c5f141df6ed93de961a0216e0a4253) --- .../client/api/BrokerServiceLookupTest.java | 103 ++++++++++++++++++ .../client/impl/BinaryProtoLookupService.java | 40 ++++++- 2 files changed, 141 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index 8597e0a87997a..4e24ffc98772d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -32,6 +32,7 @@ import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.prometheus.client.CollectorRegistry; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Field; @@ -80,6 +81,9 @@ import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.impl.BinaryProtoLookupService; +import org.apache.pulsar.client.impl.LookupService; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; @@ -102,6 +106,7 @@ import org.asynchttpclient.Response; import org.asynchttpclient.channel.DefaultKeepAliveStrategy; import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; @@ -839,6 +844,104 @@ public void testSkipSplitBundleIfOnlyOneBroker() throws Exception { } } + @Test + public void testMergeGetPartitionedMetadataRequests() throws Exception { + // Assert the lookup service is a "BinaryProtoLookupService". + final PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + final LookupService lookupService = pulsarClientImpl.getLookup(); + assertTrue(lookupService instanceof BinaryProtoLookupService); + + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + final int topicPartitions = 10; + admin.topics().createPartitionedTopic(tpName, topicPartitions); + + // Verify the request is works after merge the requests. + List> futures = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + futures.add(lookupService.getPartitionedTopicMetadata(TopicName.get(tpName))); + } + for (CompletableFuture future : futures) { + assertEquals(future.join().partitions, topicPartitions); + } + + // cleanup. + admin.topics().deletePartitionedTopic(tpName); + } + + @Test + public void testMergeLookupRequests() throws Exception { + // Assert the lookup service is a "BinaryProtoLookupService". + final PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + final LookupService lookupService = pulsarClientImpl.getLookup(); + assertTrue(lookupService instanceof BinaryProtoLookupService); + + final String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(tpName); + + // Create 1 producer and 100 consumers. + List> producers = new ArrayList<>(); + List> consumers = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + producers.add(pulsarClient.newProducer(Schema.STRING).topic(tpName).create()); + } + for (int i = 0; i < 20; i++) { + consumers.add(pulsarClient.newConsumer(Schema.STRING).topic(tpName).subscriptionName("s" + i).subscribe()); + } + + // Verify the lookup count will be smaller than before improve. + int lookupCountBeforeUnload = calculateLookupRequestCount(); + admin.namespaces().unload(TopicName.get(tpName).getNamespace()); + Awaitility.await().untilAsserted(() -> { + for (Producer p : producers) { + assertEquals(WhiteboxImpl.getInternalState(p, "state").toString(), "Ready"); + } + for (Consumer c : consumers) { + assertEquals(WhiteboxImpl.getInternalState(c, "state").toString(), "Ready"); + } + }); + int lookupCountAfterUnload = calculateLookupRequestCount(); + log.info("lookup count before unload: {}, after unload: {}", lookupCountBeforeUnload, lookupCountAfterUnload); + assertTrue(lookupCountAfterUnload < lookupCountBeforeUnload * 2, + "the lookup count should be smaller than before improve"); + + // Verify the producers and consumers is still works. + List messagesSent = new ArrayList<>(); + int index = 0; + for (Producer producer: producers) { + String message = Integer.valueOf(index++).toString(); + producer.send(message); + messagesSent.add(message); + } + HashSet messagesReceived = new HashSet<>(); + for (Consumer consumer : consumers) { + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messagesReceived.add(msg.getValue()); + } + } + assertEquals(messagesReceived.size(), producers.size()); + + // cleanup. + for (Producer producer: producers) { + producer.close(); + } + for (Consumer consumer : consumers) { + consumer.close(); + } + admin.topics().delete(tpName); + } + + private int calculateLookupRequestCount() throws Exception { + int failures = CollectorRegistry.defaultRegistry.getSampleValue("pulsar_broker_lookup_failures_total") + .intValue(); + int answers = CollectorRegistry.defaultRegistry.getSampleValue("pulsar_broker_lookup_answers_total") + .intValue(); + return failures + answers; + } + @Test(timeOut = 10000) public void testPartitionedMetadataWithDeprecatedVersion() throws Exception { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java index d5ce9213211dd..8ceb8e44975c8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BinaryProtoLookupService.java @@ -26,10 +26,12 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.SchemaSerializationException; @@ -56,6 +58,12 @@ public class BinaryProtoLookupService implements LookupService { private final String listenerName; private final int maxLookupRedirects; + private final ConcurrentHashMap>> + lookupInProgress = new ConcurrentHashMap<>(); + + private final ConcurrentHashMap> + partitionedMetadataInProgress = new ConcurrentHashMap<>(); + public BinaryProtoLookupService(PulsarClientImpl client, String serviceUrl, boolean useTls, @@ -92,7 +100,21 @@ public void updateServiceUrl(String serviceUrl) throws PulsarClientException { * @return broker-socket-address that serves given topic */ public CompletableFuture> getBroker(TopicName topicName) { - return findBroker(serviceNameResolver.resolveHost(), false, topicName, 0); + final MutableObject newFutureCreated = new MutableObject<>(); + try { + return lookupInProgress.computeIfAbsent(topicName, tpName -> { + CompletableFuture> newFuture = + findBroker(serviceNameResolver.resolveHost(), false, topicName, 0); + newFutureCreated.setValue(newFuture); + return newFuture; + }); + } finally { + if (newFutureCreated.getValue() != null) { + newFutureCreated.getValue().whenComplete((v, ex) -> { + lookupInProgress.remove(topicName, newFutureCreated.getValue()); + }); + } + } } /** @@ -100,7 +122,21 @@ public CompletableFuture> getBroker(T * */ public CompletableFuture getPartitionedTopicMetadata(TopicName topicName) { - return getPartitionedTopicMetadata(serviceNameResolver.resolveHost(), topicName); + final MutableObject newFutureCreated = new MutableObject<>(); + try { + return partitionedMetadataInProgress.computeIfAbsent(topicName, tpName -> { + CompletableFuture newFuture = + getPartitionedTopicMetadata(serviceNameResolver.resolveHost(), topicName); + newFutureCreated.setValue(newFuture); + return newFuture; + }); + } finally { + if (newFutureCreated.getValue() != null) { + newFutureCreated.getValue().whenComplete((v, ex) -> { + partitionedMetadataInProgress.remove(topicName, newFutureCreated.getValue()); + }); + } + } } private CompletableFuture> findBroker(InetSocketAddress socketAddress, From 7a033591a0d19bd62e6c34ddb9f289e45045d771 Mon Sep 17 00:00:00 2001 From: hanmz Date: Mon, 25 Sep 2023 20:53:18 +0800 Subject: [PATCH 290/494] [fix][fn] fix functions_log4j2.xml delete strategy config (#21215) --- conf/functions_log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index 6902a3acd8736..fd4042e82e82f 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -68,7 +68,7 @@ ${sys:pulsar.function.log.dir} 2 - */${sys:pulsar.function.log.file}*log.gz + ${sys:pulsar.function.log.file}*log.gz 30d @@ -101,7 +101,7 @@ ${sys:pulsar.function.log.dir} 2 - */${sys:pulsar.function.log.file}.bk*log.gz + ${sys:pulsar.function.log.file}.bk*log.gz 30d From 96e7c66c9242ce933673960f840a56908ce9aafe Mon Sep 17 00:00:00 2001 From: Guangning E Date: Sun, 24 Sep 2023 20:22:59 +0800 Subject: [PATCH 291/494] [fix][broker]Fixed produce and consume when anonymousUserRole enabled (#21237) --- .../pulsar/broker/service/ServerCnx.java | 19 +++++++--- .../pulsar/broker/service/ServerCnxTest.java | 35 +++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index 3fc1e96a57d96..c34085bcd0cf5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -988,7 +988,6 @@ protected void handleConnect(CommandConnect connect) { try { byte[] authData = connect.hasAuthData() ? connect.getAuthData() : emptyArray; AuthData clientData = AuthData.of(authData); - // init authentication if (connect.hasAuthMethodName()) { authMethod = connect.getAuthMethodName(); @@ -1047,10 +1046,22 @@ protected void handleConnect(CommandConnect connect) { .getAuthenticationService() .getAuthenticationProvider(originalAuthMethod); + /** + * When both the broker and the proxy are configured with anonymousUserRole + * if the client does not configure an authentication method + * the proxy side will set the value of anonymousUserRole to clientAuthRole when it creates a connection + * and the value of clientAuthMethod will be none. + * Similarly, should also set the value of authRole to anonymousUserRole on the broker side. + */ if (originalAuthenticationProvider == null) { - throw new AuthenticationException( - String.format("Can't find AuthenticationProvider for original role" - + " using auth method [%s] is not available", originalAuthMethod)); + authRole = getBrokerService().getAuthenticationService().getAnonymousUserRole() + .orElseThrow(() -> + new AuthenticationException("No anonymous role, and can't find " + + "AuthenticationProvider for original role using auth method " + + "[" + originalAuthMethod + "] is not available")); + originalPrincipal = authRole; + completeConnect(clientProtocolVersion, clientVersion); + return; } originalAuthDataCopy = AuthData.of(connect.getOriginalAuthData().getBytes()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 2ea5e28880bf8..5fd4881981365 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -506,6 +506,41 @@ public void testConnectCommandWithPassingOriginalAuthData() throws Exception { channel.finish(); } + @Test(timeOut = 30000) + public void testConnectCommandWithPassingOriginalAuthDataAndSetAnonymousUserRole() throws Exception { + AuthenticationService authenticationService = mock(AuthenticationService.class); + AuthenticationProvider authenticationProvider = new MockAuthenticationProvider(); + String authMethodName = authenticationProvider.getAuthMethodName(); + + String anonymousUserRole = "admin"; + when(brokerService.getAuthenticationService()).thenReturn(authenticationService); + when(authenticationService.getAuthenticationProvider(authMethodName)).thenReturn(authenticationProvider); + when(authenticationService.getAnonymousUserRole()).thenReturn(Optional.of(anonymousUserRole)); + svcConfig.setAuthenticationEnabled(true); + svcConfig.setAuthenticateOriginalAuthData(true); + svcConfig.setProxyRoles(Collections.singleton("pass.proxy")); + svcConfig.setAnonymousUserRole(anonymousUserRole); + + resetChannel(); + assertTrue(channel.isActive()); + assertEquals(serverCnx.getState(), State.Start); + + // When both the proxy and the broker set the anonymousUserRole option + // the proxy will use anonymousUserRole to delegate the client's role when connecting. + ByteBuf clientCommand = Commands.newConnect(authMethodName, "pass.proxy", 1, null, + null, anonymousUserRole, null, null); + channel.writeInbound(clientCommand); + + Object response1 = getResponse(); + assertTrue(response1 instanceof CommandConnected); + assertEquals(serverCnx.getState(), State.Connected); + assertEquals(serverCnx.getAuthRole(), anonymousUserRole); + assertEquals(serverCnx.getPrincipal(), anonymousUserRole); + assertEquals(serverCnx.getOriginalPrincipal(), anonymousUserRole); + assertTrue(serverCnx.isActive()); + channel.finish(); + } + @Test(timeOut = 30000) public void testConnectCommandWithPassingOriginalPrincipal() throws Exception { AuthenticationService authenticationService = mock(AuthenticationService.class); From 8cb6af6ae41c3387c767d1dcf67533a5b1c166a7 Mon Sep 17 00:00:00 2001 From: Asaf Mesika Date: Sun, 24 Sep 2023 06:43:40 +0300 Subject: [PATCH 292/494] [fix][broker] Fixed reset for AggregatedNamespaceStats (#21225) --- pulsar-broker/pom.xml | 1 + .../prometheus/AggregatedNamespaceStats.java | 24 +++- .../AggregatedNamespaceStatsTest.java | 103 +++++++++++++++++- 3 files changed, 124 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index cb12cdfedfe54..8149c02e5ac9d 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -568,6 +568,7 @@ protobuf-maven-plugin ${protobuf-maven-plugin.version} + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} true diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java index 9fe5588044d2f..d0dc4fe2a7e7d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStats.java @@ -34,7 +34,7 @@ public class AggregatedNamespaceStats { public double throughputIn; public double throughputOut; - public long messageAckRate; + public double messageAckRate; public long bytesInCounter; public long msgInCounter; public long bytesOutCounter; @@ -64,7 +64,7 @@ public class AggregatedNamespaceStats { long compactionCompactedEntriesCount; long compactionCompactedEntriesSize; StatsBuckets compactionLatencyBuckets = new StatsBuckets(CompactionRecord.WRITE_LATENCY_BUCKETS_USEC); - int delayedMessageIndexSizeInBytes; + long delayedMessageIndexSizeInBytes; Map bucketDelayedIndexStats = new HashMap<>(); @@ -182,14 +182,34 @@ public void reset() { rateOut = 0; throughputIn = 0; throughputOut = 0; + messageAckRate = 0; + bytesInCounter = 0; + msgInCounter = 0; + + bytesOutCounter = 0; + msgOutCounter = 0; msgBacklog = 0; msgDelayed = 0; + ongoingTxnCount = 0; + abortedTxnCount = 0; + committedTxnCount = 0; + backlogQuotaLimit = 0; backlogQuotaLimitTime = -1; replicationStats.clear(); subscriptionStats.clear(); + + compactionRemovedEventCount = 0; + compactionSucceedCount = 0; + compactionFailedCount = 0; + compactionDurationTimeInMills = 0; + compactionReadThroughput = 0; + compactionWriteThroughput = 0; + compactionCompactedEntriesCount = 0; + compactionCompactedEntriesSize = 0; + delayedMessageIndexSizeInBytes = 0; bucketDelayedIndexStats.clear(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStatsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStatsTest.java index b5933f9ecf529..0e12d75f74fa0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStatsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/stats/prometheus/AggregatedNamespaceStatsTest.java @@ -20,10 +20,12 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; - +import java.util.HashMap; +import org.apache.bookkeeper.mledger.util.StatsBuckets; +import org.apache.pulsar.common.policies.data.stats.TopicMetricBean; import org.testng.annotations.Test; -@Test(groups = "broker") +@Test(groups = {"broker"}) public class AggregatedNamespaceStatsTest { @Test @@ -157,4 +159,101 @@ public void testSimpleAggregation() { assertEquals(nsSubStats.unackedMessages, 2); } + + @Test + public void testReset() { + AggregatedNamespaceStats stats = new AggregatedNamespaceStats(); + stats.topicsCount = 8; + stats.subscriptionsCount = 3; + stats.producersCount = 1; + stats.consumersCount = 8; + stats.rateIn = 1.3; + stats.rateOut = 3.5; + stats.throughputIn = 3.2; + stats.throughputOut = 5.8; + stats.messageAckRate = 12; + stats.bytesInCounter = 1234; + stats.msgInCounter = 3889; + stats.bytesOutCounter = 89775; + stats.msgOutCounter = 28983; + stats.msgBacklog = 39; + stats.msgDelayed = 31; + + stats.ongoingTxnCount = 87; + stats.abortedTxnCount = 74; + stats.committedTxnCount = 34; + + stats.backlogQuotaLimit = 387; + stats.backlogQuotaLimitTime = 8771; + + stats.replicationStats = new HashMap<>(); + stats.replicationStats.put("r", new AggregatedReplicationStats()); + + stats.subscriptionStats = new HashMap<>(); + stats.subscriptionStats.put("r", new AggregatedSubscriptionStats()); + + stats.compactionRemovedEventCount = 124; + stats.compactionSucceedCount = 487; + stats.compactionFailedCount = 84857; + stats.compactionDurationTimeInMills = 2384; + stats.compactionReadThroughput = 355423; + stats.compactionWriteThroughput = 23299; + stats.compactionCompactedEntriesCount = 37522; + stats.compactionCompactedEntriesSize = 8475; + + stats.compactionLatencyBuckets = new StatsBuckets(5); + stats.compactionLatencyBuckets.addValue(3); + + stats.delayedMessageIndexSizeInBytes = 45223; + + stats.bucketDelayedIndexStats = new HashMap<>(); + stats.bucketDelayedIndexStats.put("t", new TopicMetricBean()); + + stats.reset(); + + assertEquals(stats.bytesOutCounter, 0); + assertEquals(stats.topicsCount, 0); + assertEquals(stats.subscriptionsCount, 0); + assertEquals(stats.producersCount, 0); + assertEquals(stats.consumersCount, 0); + assertEquals(stats.rateIn, 0); + assertEquals(stats.rateOut, 0); + assertEquals(stats.throughputIn, 0); + assertEquals(stats.throughputOut, 0); + assertEquals(stats.messageAckRate, 0); + assertEquals(stats.bytesInCounter, 0); + assertEquals(stats.msgInCounter, 0); + assertEquals(stats.bytesOutCounter, 0); + assertEquals(stats.msgOutCounter, 0); + + assertEquals(stats.managedLedgerStats.storageSize, 0); + + assertEquals(stats.msgBacklog, 0); + assertEquals(stats.msgDelayed, 0); + + assertEquals(stats.ongoingTxnCount, 0); + assertEquals(stats.abortedTxnCount, 0); + assertEquals(stats.committedTxnCount, 0); + + assertEquals(stats.backlogQuotaLimit, 0); + assertEquals(stats.backlogQuotaLimitTime, -1); + + assertEquals(stats.replicationStats.size(), 0); + assertEquals(stats.subscriptionStats.size(), 0); + + assertEquals(stats.compactionRemovedEventCount, 0); + assertEquals(stats.compactionSucceedCount, 0); + assertEquals(stats.compactionFailedCount, 0); + assertEquals(stats.compactionDurationTimeInMills, 0); + assertEquals(stats.compactionReadThroughput, 0); + assertEquals(stats.compactionWriteThroughput, 0); + assertEquals(stats.compactionCompactedEntriesCount, 0); + assertEquals(stats.compactionCompactedEntriesSize, 0); + + assertEquals(stats.compactionLatencyBuckets.getSum(), 0); + + assertEquals(stats.delayedMessageIndexSizeInBytes, 0); + assertEquals(stats.bucketDelayedIndexStats.size(), 0); + } + } From e28ef97470aca72efcce494af67039bfc55763c4 Mon Sep 17 00:00:00 2001 From: hanmz Date: Thu, 21 Sep 2023 10:04:41 +0800 Subject: [PATCH 293/494] [fix][broker] replicator leak when removeReplicator in NonPersistentTopic (#21205) --- .../pulsar/broker/service/nonpersistent/NonPersistentTopic.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index c4ace2bebb646..cd09f18736814 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -637,6 +637,7 @@ CompletableFuture removeReplicator(String remoteCluster) { replicators.get(remoteCluster).disconnect().thenRun(() -> { log.info("[{}] Successfully removed replicator {}", name, remoteCluster); + replicators.remove(remoteCluster); }).exceptionally(e -> { log.error("[{}] Failed to close replication producer {} {}", topic, name, e.getMessage(), e); From 298a320cd31db0ef7d065aa0c0304967d5114096 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Sun, 17 Sep 2023 21:44:05 +0800 Subject: [PATCH 294/494] [fix][auto-recovery] Fix metadata store deadlock due to BookkeeperInternalCallbacks.Processor (#21159) --- pom.xml | 14 ++ pulsar-metadata/pom.xml | 21 ++ .../AbstractHierarchicalLedgerManager.java | 9 +- .../replication/AuditorPeriodicCheckTest.java | 219 ++++++++++++++++++ 4 files changed, 259 insertions(+), 4 deletions(-) create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java diff --git a/pom.xml b/pom.xml index 5bf2d2df8453d..6b6d98d865b85 100644 --- a/pom.xml +++ b/pom.xml @@ -1472,6 +1472,20 @@ flexible messaging model and an intuitive client API. + + + org.apache.bookkeeper + bookkeeper-common + ${bookkeeper.version} + test + tests + + + com.fasterxml.jackson.core + * + + + diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 25c1010a8460a..421d461d3b0bf 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -64,6 +64,27 @@ test + + org.apache.zookeeper + zookeeper + tests + test + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + io.netty + netty-tcnative + + + + org.xerial.snappy diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractHierarchicalLedgerManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractHierarchicalLedgerManager.java index a33c8761cba9e..4db7f4798c309 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractHierarchicalLedgerManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/AbstractHierarchicalLedgerManager.java @@ -206,10 +206,11 @@ protected void asyncProcessLedgersInSingleNode( mcb = new BookkeeperInternalCallbacks.MultiCallback(activeLedgers.size(), finalCb, ctx, successRc, failureRc); // start loop over all ledgers - for (Long ledger : activeLedgers) { - processor.process(ledger, mcb); - } - + scheduler.submit(() -> { + for (Long ledger : activeLedgers) { + processor.process(ledger, mcb); + } + }); }).exceptionally(ex -> { finalCb.processResult(failureRc, null, ctx); return null; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java new file mode 100644 index 0000000000000..c761d46c62266 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotSame; +import io.netty.buffer.ByteBuf; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.bookie.TestBookieImpl; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataBookieDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +/** + * This test verifies that the period check on the auditor + * will pick up on missing data in the client. + */ +public class AuditorPeriodicCheckTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorPeriodicCheckTest.class); + + private MetadataBookieDriver driver; + private HashMap auditorElectors = new HashMap(); + + private static final int CHECK_INTERVAL = 1; // run every second + + public AuditorPeriodicCheckTest() throws Exception { + super(3); + baseConf.setPageLimit(1); // to make it easy to push ledger out of cache + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeTest + @Override + public void setUp() throws Exception { + super.setUp(); + + for (int i = 0; i < numBookies; i++) { + ServerConfiguration conf = new ServerConfiguration(confByIndex(i)); + conf.setAuditorPeriodicCheckInterval(CHECK_INTERVAL); + conf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + + String addr = addressByIndex(i).toString(); + + AuditorElector auditorElector = new AuditorElector(addr, conf); + auditorElectors.put(addr, auditorElector); + auditorElector.start(); + if (LOG.isDebugEnabled()) { + LOG.debug("Starting Auditor Elector"); + } + } + + URI uri = URI.create(confByIndex(0).getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver = MetadataDrivers.getBookieDriver(uri); + ServerConfiguration serverConfiguration = new ServerConfiguration(confByIndex(0)); + serverConfiguration.setMetadataServiceUri( + serverConfiguration.getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE); + } + + @AfterTest + @Override + public void tearDown() throws Exception { + if (null != driver) { + driver.close(); + } + + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + super.tearDown(); + } + + private BookieId replaceBookieWithWriteFailingBookie(LedgerHandle lh) throws Exception { + int bookieIdx = -1; + Long entryId = lh.getLedgerMetadata().getAllEnsembles().firstKey(); + List curEnsemble = lh.getLedgerMetadata().getAllEnsembles().get(entryId); + + // Identify a bookie in the current ledger ensemble to be replaced + BookieId replacedBookie = null; + for (int i = 0; i < numBookies; i++) { + if (curEnsemble.contains(addressByIndex(i))) { + bookieIdx = i; + replacedBookie = addressByIndex(i); + break; + } + } + assertNotSame("Couldn't find ensemble bookie in bookie list", -1, bookieIdx); + + LOG.info("Killing bookie " + addressByIndex(bookieIdx)); + ServerConfiguration conf = killBookie(bookieIdx); + Bookie writeFailingBookie = new TestBookieImpl(conf) { + @Override + public void addEntry(ByteBuf entry, boolean ackBeforeSync, WriteCallback cb, + Object ctx, byte[] masterKey) + throws IOException, BookieException { + try { + LOG.info("Failing write to entry "); + // sleep a bit so that writes to other bookies succeed before + // the client hears about the failure on this bookie. If the + // client gets ack-quorum number of acks first, it won't care + // about any failures and won't reform the ensemble. + Thread.sleep(100); + throw new IOException(); + } catch (InterruptedException ie) { + // ignore, only interrupted if shutting down, + // and an exception would spam the logs + Thread.currentThread().interrupt(); + } + } + }; + startAndAddBookie(conf, writeFailingBookie); + return replacedBookie; + } + + /* + * Validates that the periodic ledger check will fix entries with a failed write. + */ + @Test + public void testFailedWriteRecovery() throws Exception { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + underReplicationManager.disableLedgerReplication(); + + LedgerHandle lh = bkc.createLedger(2, 2, 1, DigestType.CRC32, "passwd".getBytes()); + + // kill one of the bookies and replace it with one that rejects write; + // This way we get into the under replication state + BookieId replacedBookie = replaceBookieWithWriteFailingBookie(lh); + + // Write a few entries; this should cause under replication + byte[] data = "foobar".getBytes(); + data = "foobar".getBytes(); + lh.addEntry(data); + lh.addEntry(data); + lh.addEntry(data); + + lh.close(); + + // enable under replication detection and wait for it to report + // under replicated ledger + underReplicationManager.enableLedgerReplication(); + long underReplicatedLedger = -1; + for (int i = 0; i < 5; i++) { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger != -1) { + break; + } + Thread.sleep(CHECK_INTERVAL * 1000); + } + assertEquals("Ledger should be under replicated", lh.getId(), underReplicatedLedger); + + // now start the replication workers + List l = new ArrayList(); + for (int i = 0; i < numBookies; i++) { + ReplicationWorker rw = new ReplicationWorker(confByIndex(i), NullStatsLogger.INSTANCE); + rw.start(); + l.add(rw); + } + underReplicationManager.close(); + + // Wait for ensemble to change after replication + Thread.sleep(3000); + for (ReplicationWorker rw : l) { + rw.shutdown(); + } + + // check that ensemble has changed and the bookie that rejected writes has + // been replaced in the ensemble + LedgerHandle newLh = bkc.openLedger(lh.getId(), DigestType.CRC32, "passwd".getBytes()); + for (Map.Entry> e : + newLh.getLedgerMetadata().getAllEnsembles().entrySet()) { + List ensemble = e.getValue(); + assertFalse("Ensemble hasn't been updated", ensemble.contains(replacedBookie)); + } + newLh.close(); + } +} From 44db9a0139ed39108c0cf4c2e62a1c4c9995cef4 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Wed, 13 Sep 2023 17:31:28 +0800 Subject: [PATCH 295/494] [fix] [bookie] Fix RocksDB configuration (#21157) --- conf/entry_location_rocksdb.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/entry_location_rocksdb.conf b/conf/entry_location_rocksdb.conf index 31bd58506ef75..42d916ded378f 100644 --- a/conf/entry_location_rocksdb.conf +++ b/conf/entry_location_rocksdb.conf @@ -52,6 +52,8 @@ max_bytes_for_level_base=268435456 # set by jni: options.setTargetFileSizeBase target_file_size_base=67108864 + # set by jni: options.setLevelCompactionDynamicLevelBytes + level_compaction_dynamic_level_bytes=true [TableOptions/BlockBasedTable "default"] # set by jni: tableOptions.setBlockSize @@ -66,5 +68,3 @@ filter_policy=rocksdb.BloomFilter:10:false # set by jni: tableOptions.setCacheIndexAndFilterBlocks cache_index_and_filter_blocks=true - # set by jni: options.setLevelCompactionDynamicLevelBytes - level_compaction_dynamic_level_bytes=true \ No newline at end of file From dae4e013c757e1da3dada2af3c34ee15fd052b55 Mon Sep 17 00:00:00 2001 From: hrzzzz <64506104+hrzzzz@users.noreply.github.com> Date: Wed, 27 Sep 2023 15:45:20 +0800 Subject: [PATCH 296/494] [fix][txn] fix the consumer stuck due to deduplicated messages in pending ack state (#21177) --- .../service/AbstractBaseDispatcher.java | 11 ++-- .../client/impl/TransactionEndToEndTest.java | 59 +++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java index 171cbc3bf4b9f..e778955f8f5e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractBaseDispatcher.java @@ -213,12 +213,7 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i this.filterAcceptedMsgs.add(entryMsgCnt); } - totalEntries++; int batchSize = msgMetadata.getNumMessagesInBatch(); - totalMessages += batchSize; - totalBytes += metadataAndPayload.readableBytes(); - totalChunkedMessages += msgMetadata.hasChunkId() ? 1 : 0; - batchSizes.setBatchSize(i, batchSize); long[] ackSet = null; if (indexesAcks != null && cursor != null) { PositionImpl position = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); @@ -262,6 +257,12 @@ public int filterEntriesForConsumer(@Nullable MessageMetadata[] metadataArray, i } } + totalEntries++; + totalMessages += batchSize; + totalBytes += metadataAndPayload.readableBytes(); + totalChunkedMessages += msgMetadata.hasChunkId() ? 1 : 0; + batchSizes.setBatchSize(i, batchSize); + BrokerInterceptor interceptor = subscription.interceptor(); if (null != interceptor) { // keep for compatibility if users has implemented the old interface diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java index 34cc3bc1ca526..348fb04b7dd23 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TransactionEndToEndTest.java @@ -254,6 +254,65 @@ private void testFilterMsgsInPendingAckStateWhenConsumerDisconnect(boolean enabl Assert.assertEquals(receiveCounter, count / 2); } + @Test + private void testMsgsInPendingAckStateWouldNotGetTheConsumerStuck() throws Exception { + final String topicName = NAMESPACE1 + "/testMsgsInPendingAckStateWouldNotGetTheConsumerStuck"; + final String subscription = "test"; + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.INT32) + .topic(topicName) + .create(); + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.INT32) + .topic(topicName) + .subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + int numStep1Receive = 2, numStep2Receive = 2, numStep3Receive = 2; + int numTotalMessage = numStep1Receive + numStep2Receive + numStep3Receive; + + for (int i = 0; i < numTotalMessage; i++) { + producer.send(i); + } + + Transaction step1Txn = getTxn(); + Transaction step2Txn = getTxn(); + + // Step 1, try to consume some messages but do not commit the transaction + for (int i = 0; i < numStep1Receive; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), step1Txn).get(); + } + + // Step 2, try to consume some messages and commit the transaction + for (int i = 0; i < numStep2Receive; i++) { + consumer.acknowledgeAsync(consumer.receive().getMessageId(), step2Txn).get(); + } + + // commit step2Txn + step2Txn.commit().get(); + + // close and re-create consumer + consumer.close(); + @Cleanup + Consumer consumer2 = pulsarClient.newConsumer(Schema.INT32) + .topic(topicName) + .receiverQueueSize(numStep3Receive) + .subscriptionName(subscription) + .subscriptionType(SubscriptionType.Shared) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + // Step 3, try to consume the rest messages and should receive all of them + for (int i = 0; i < numStep3Receive; i++) { + // should get the message instead of timeout + Message msg = consumer2.receive(3, TimeUnit.SECONDS); + Assert.assertEquals(msg.getValue(), numStep1Receive + numStep2Receive + i); + } + } + @Test(dataProvider="enableBatch") private void produceCommitTest(boolean enableBatch) throws Exception { @Cleanup From 62acc2b01279769fa9cbd1c9952172ba8b15eccd Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 30 Sep 2023 11:27:36 +0300 Subject: [PATCH 297/494] [fix][sec] Add OWASP Dependency Check suppressions (#21281) (cherry picked from commit 1bf7371b6d33c4e015d006e547b393b97686ff20) # Conflicts: # src/owasp-dependency-check-suppressions.xml --- src/owasp-dependency-check-suppressions.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 311204ac37085..1a82702105bfb 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -404,4 +404,16 @@ CVE-2020-8908 + + + CVE-2023-37475 + + + + CVE-2023-4586 + From fbe2c707bb8de2e95183f7d4f3dfa681bb3949bb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 30 Sep 2023 12:14:11 +0300 Subject: [PATCH 298/494] [fix][build] Upgrade Lombok to 1.18.30 to support compiling with JDK21 (#21278) (cherry picked from commit 682eb36a76746fd47ce57dd8915e3d4603aa039e) # Conflicts: # pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6b6d98d865b85..5b8170cec3722 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ flexible messaging model and an intuitive client API. 0.9.1 2.1.0 3.24.2 - 1.18.26 + 1.18.30 1.3.2 2.3.1 1.2.0 From 0bfa54fd3787f5fef1998a64a514db5d24c95ef2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 7 Oct 2023 05:26:48 +0300 Subject: [PATCH 299/494] [fix][ml] Fix thread safe issue with RangeCache.put and RangeCache.clear (#21302) (cherry picked from commit 70d086f8f35d36800059d0d68e13d0ca017bf233) --- .../bookkeeper/mledger/util/RangeCache.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java index 7599e2cc1874f..d34857e5e5177 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/util/RangeCache.java @@ -27,7 +27,6 @@ import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicLong; -import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.tuple.Pair; /** @@ -74,13 +73,18 @@ public RangeCache(Weighter weighter, TimestampExtractor timestampE * @return whether the entry was inserted in the cache */ public boolean put(Key key, Value value) { - MutableBoolean flag = new MutableBoolean(); - entries.computeIfAbsent(key, (k) -> { - size.addAndGet(weighter.getSize(value)); - flag.setValue(true); - return value; - }); - return flag.booleanValue(); + // retain value so that it's not released before we put it in the cache and calculate the weight + value.retain(); + try { + if (entries.putIfAbsent(key, value) == null) { + size.addAndGet(weighter.getSize(value)); + return true; + } else { + return false; + } + } finally { + value.release(); + } } public boolean exists(Key key) { @@ -242,7 +246,6 @@ public synchronized Pair clear() { value.release(); } - entries.clear(); size.getAndAdd(-removedSize); return Pair.of(removedCount, removedSize); } From f243925dfedb5c2670a5f3f1378807d572db891d Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 29 Sep 2023 00:12:16 +0800 Subject: [PATCH 300/494] [improve] [broker] Print warn log if ssl handshake error & print ledger id when switch ledger (#21201) ### Modifications - Print a warning log if the SSL handshake error - Print ledger ID when switching ledger (cherry picked from commit 8485d68197300dca83b7b443c8aebc4418a85e4b) --- .../bookkeeper/mledger/impl/ManagedCursorImpl.java | 3 ++- .../bookkeeper/mledger/impl/ManagedLedgerImpl.java | 6 ++++-- .../org/apache/pulsar/client/impl/ClientCnx.java | 13 +++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 4500d27785240..9063602dd703d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1266,7 +1266,8 @@ protected void internalResetCursor(PositionImpl proposedReadPosition, newReadPosition = proposedReadPosition; } - log.info("[{}] Initiate reset readPosition to {} on cursor {}", ledger.getName(), newReadPosition, name); + log.info("[{}] Initiate reset readPosition from {} to {} on cursor {}", ledger.getName(), readPosition, + newReadPosition, name); synchronized (pendingMarkDeleteOps) { if (!RESET_CURSOR_IN_PROGRESS_UPDATER.compareAndSet(this, FALSE, TRUE)) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 4991080d38cbe..0f65a1a089c1f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -529,7 +529,8 @@ public void operationFailed(MetaStoreException e) { return; } - log.info("[{}] Created ledger {}", name, lh.getId()); + log.info("[{}] Created ledger {} after closed {}", name, lh.getId(), + currentLedger == null ? "null" : currentLedger.getId()); STATE_UPDATER.set(this, State.LedgerOpened); updateLastLedgerCreatedTimeAndScheduleRolloverTask(); currentLedger = lh; @@ -1770,7 +1771,8 @@ public void skipNonRecoverableLedger(long ledgerId){ synchronized void createLedgerAfterClosed() { if (isNeededCreateNewLedgerAfterCloseLedger()) { - log.info("[{}] Creating a new ledger after closed", name); + log.info("[{}] Creating a new ledger after closed {}", name, + currentLedger == null ? "null" : currentLedger.getId()); STATE_UPDATER.set(this, State.CreatingLedger); this.lastLedgerCreationInitiationTimestamp = System.currentTimeMillis(); mbean.startDataLedgerCreateOp(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 115c71307c4f2..f3e8b2354b344 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -30,6 +30,7 @@ import io.netty.channel.EventLoopGroup; import io.netty.channel.unix.Errors.NativeIoException; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; +import io.netty.handler.ssl.SslHandshakeCompletionEvent; import io.netty.util.concurrent.Promise; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -1311,6 +1312,18 @@ public void close() { } } + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof SslHandshakeCompletionEvent) { + SslHandshakeCompletionEvent sslHandshakeCompletionEvent = (SslHandshakeCompletionEvent) evt; + if (sslHandshakeCompletionEvent.cause() != null) { + log.warn("{} Got ssl handshake exception {}", ctx.channel(), + sslHandshakeCompletionEvent); + } + } + ctx.fireUserEventTriggered(evt); + } + protected void closeWithException(Throwable e) { if (ctx != null) { connectionFuture.completeExceptionally(e); From 38c3f0c019fdf3cb8f1dcf0c739226ebea777bc7 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 28 Sep 2023 23:45:39 +0800 Subject: [PATCH 301/494] [fix] [client] fix reader.hasMessageAvailable return false when incoming queue is not empty (#21259) Reproduce steps: - Create a reader. - Reader pulls messages into `incoming queue`, do not call `reader.readNext` now. - Trim ledger task will delete the ledgers, then there is no in the topic. - Now, you can get messages if you call `reader.readNext`, but the method `reader.hasMessageAvailable` return `false` Note: the similar issue of `MultiTopicsConsumerImpl` has been fixed by https://github.com/apache/pulsar/pull/13332, current PR only trying to fix the issue of `ConsumerImpl`. Make `reader.hasMessageAvailable` return `true` when `incoming queue` is not empty. (cherry picked from commit 6d82b09128f46fdcb27021560d773fac15d66a48) --- .../api/NonDurableSubscriptionTest.java | 89 ++++++++++++++++++- .../pulsar/client/impl/ConsumerImpl.java | 4 + 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java index 52b1498ef7de9..6762b7d8a045e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java @@ -19,24 +19,34 @@ package org.apache.pulsar.client.api; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; -import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertNull; import static org.testng.AssertJUnit.assertTrue; import java.lang.reflect.Field; import java.util.UUID; +import java.lang.reflect.Method; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.PulsarChannelInitializer; import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentSubscription; import org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.api.proto.CommandFlow; +import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -45,7 +55,7 @@ @Test(groups = "broker-api") @Slf4j -public class NonDurableSubscriptionTest extends ProducerConsumerBase { +public class NonDurableSubscriptionTest extends ProducerConsumerBase { private final AtomicInteger numFlow = new AtomicInteger(0); @@ -297,4 +307,79 @@ public void testFlowCountForMultiTopics() throws Exception { assertEquals(numFlow.get(), numPartitions); } + + private void trimLedgers(final String tpName) { + // Wait for topic loading. + org.awaitility.Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + assertNotNull(persistentTopic); + }); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + CompletableFuture trimLedgersTask = new CompletableFuture<>(); + ml.trimConsumedLedgersInBackground(trimLedgersTask); + trimLedgersTask.join(); + } + + private void switchLedgerManually(final String tpName) throws Exception { + Method ledgerClosed = + ManagedLedgerImpl.class.getDeclaredMethod("ledgerClosed", new Class[]{LedgerHandle.class}); + Method createLedgerAfterClosed = + ManagedLedgerImpl.class.getDeclaredMethod("createLedgerAfterClosed", new Class[0]); + ledgerClosed.setAccessible(true); + createLedgerAfterClosed.setAccessible(true); + + // Wait for topic create. + org.awaitility.Awaitility.await().untilAsserted(() -> { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + assertNotNull(persistentTopic); + }); + + // Switch ledger. + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(tpName, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + LedgerHandle currentLedger1 = WhiteboxImpl.getInternalState(ml, "currentLedger"); + ledgerClosed.invoke(ml, new Object[]{currentLedger1}); + createLedgerAfterClosed.invoke(ml, new Object[0]); + Awaitility.await().untilAsserted(() -> { + LedgerHandle currentLedger2 = WhiteboxImpl.getInternalState(ml, "currentLedger"); + assertNotEquals(currentLedger1.getId(), currentLedger2.getId()); + }); + } + + @Test + public void testTrimLedgerIfNoDurableCursor() throws Exception { + final String nonDurableCursor = "non-durable-cursor"; + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + Reader reader = pulsarClient.newReader(Schema.STRING).topic(topicName).receiverQueueSize(1) + .subscriptionName(nonDurableCursor).startMessageId(MessageIdImpl.earliest).create(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + MessageIdImpl msgSent = (MessageIdImpl) producer.send("1"); + + // Trigger switch ledger. + // Trigger a trim ledgers task, and verify trim ledgers successful. + switchLedgerManually(topicName); + trimLedgers(topicName); + + // Since there is one message in the incoming queue, so the method "reader.hasMessageAvailable" should return + // true. + boolean hasMessageAvailable = reader.hasMessageAvailable(); + Message msgReceived = reader.readNext(2, TimeUnit.SECONDS); + if (msgReceived == null) { + assertFalse(hasMessageAvailable); + } else { + log.info("receive msg: {}", msgReceived.getValue()); + assertTrue(hasMessageAvailable); + assertEquals(msgReceived.getValue(), "1"); + } + + // cleanup. + reader.close(); + producer.close(); + admin.topics().delete(topicName); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 3a542e6610419..feb9d53b607a7 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -2318,6 +2318,10 @@ public boolean hasMessageAvailable() throws PulsarClientException { public CompletableFuture hasMessageAvailableAsync() { final CompletableFuture booleanFuture = new CompletableFuture<>(); + if (incomingMessages != null && !incomingMessages.isEmpty()) { + return CompletableFuture.completedFuture(true); + } + // we haven't read yet. use startMessageId for comparison if (lastDequeuedMessageId == MessageId.earliest) { // if we are starting from latest, we should seek to the actual last message first. From eb2144e171248bed26bbc252e74276f642d30077 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 28 Sep 2023 00:19:20 +0800 Subject: [PATCH 302/494] [improve] [broker] Not close the socket if lookup failed caused by bundle unloading or metadata ex (#21211) ### Motivation **Background**: The Pulsar client will close the socket if it receives a ServiceNotReady error when doing a lookup. Closing the socket causes the other consumer or producer to reconnect and does not make the lookup more efficient. There are two cases that should be improved: - If the broker gets a metadata read/write error, the broker responds with a `ServiceNotReady` error, but it should respond with a `MetadataError` - If the topic is unloading, the broker responds with a `ServiceNotReady` error. ### Modifications - Respond to the client with a `MetadataError` if the broker gets a metadata read/write error. - Respond to the client with a `MetadataError` if the topic is unloading (cherry picked from commit 09a17203702a032274a56d7d0ff03c9e32b4529f) --- .../pulsar/broker/lookup/TopicLookupBase.java | 46 ++++---- .../broker/namespace/ServiceUnitUtils.java | 2 +- .../pulsar/broker/web/PulsarWebResource.java | 1 + .../LeaderElectionServiceTest.java | 6 +- .../client/api/BrokerServiceLookupTest.java | 103 ++++++++++++++++++ 5 files changed, 132 insertions(+), 26 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index bd70201cba55d..a8dda145f6b84 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -26,7 +26,6 @@ import java.net.URISyntaxException; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import javax.ws.rs.Encoded; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; @@ -48,6 +47,7 @@ import org.apache.pulsar.common.policies.data.TopicOperation; import org.apache.pulsar.common.util.Codec; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataStoreException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -318,35 +318,37 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe requestId, shouldRedirectThroughServiceUrl(conf, lookupData))); } }).exceptionally(ex -> { - if (ex instanceof CompletionException && ex.getCause() instanceof IllegalStateException) { - log.info("Failed to lookup {} for topic {} with error {}", clientAppId, - topicName.toString(), ex.getCause().getMessage()); - } else { - log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, - topicName.toString(), ex.getMessage(), ex); - } - lookupfuture.complete( - newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId)); - return null; - }); + handleLookupError(lookupfuture, topicName.toString(), clientAppId, requestId, ex); + return null; + }); } - }).exceptionally(ex -> { - if (ex instanceof CompletionException && ex.getCause() instanceof IllegalStateException) { - log.info("Failed to lookup {} for topic {} with error {}", clientAppId, topicName.toString(), - ex.getCause().getMessage()); - } else { - log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName.toString(), - ex.getMessage(), ex); - } - - lookupfuture.complete(newLookupErrorResponse(ServerError.ServiceNotReady, ex.getMessage(), requestId)); + handleLookupError(lookupfuture, topicName.toString(), clientAppId, requestId, ex); return null; }); return lookupfuture; } + private static void handleLookupError(CompletableFuture lookupFuture, String topicName, String clientAppId, + long requestId, Throwable ex){ + final Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); + final String errorMsg = unwrapEx.getMessage(); + if (unwrapEx instanceof IllegalStateException) { + // Current broker still hold the bundle's lock, but the bundle is being unloading. + log.info("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); + lookupFuture.complete(newLookupErrorResponse(ServerError.MetadataError, errorMsg, requestId)); + } else if (unwrapEx instanceof MetadataStoreException){ + // Load bundle ownership or acquire lock failed. + // Differ with "IllegalStateException", print warning log. + log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); + lookupFuture.complete(newLookupErrorResponse(ServerError.MetadataError, errorMsg, requestId)); + } else { + log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); + lookupFuture.complete(newLookupErrorResponse(ServerError.ServiceNotReady, errorMsg, requestId)); + } + } + protected TopicName getTopicName(String topicDomain, String tenant, String cluster, String namespace, @Encoded String encodedTopic) { String decodedName = Codec.decode(encodedTopic); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/ServiceUnitUtils.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/ServiceUnitUtils.java index c86aac5316fb9..432aa29798ebd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/ServiceUnitUtils.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/ServiceUnitUtils.java @@ -36,7 +36,7 @@ public final class ServiceUnitUtils { */ private static final String OWNER_INFO_ROOT = "/namespace"; - static String path(NamespaceBundle suname) { + public static String path(NamespaceBundle suname) { // The ephemeral node path for new namespaces should always have bundle name appended return OWNER_INFO_ROOT + "/" + suname.toString(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java index bfb94aa7740a6..bdb052d7b9e7c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/PulsarWebResource.java @@ -528,6 +528,7 @@ protected static CompletableFuture getClusterDataIfDifferentCluster pulsar.getPulsarResources().getClusterResources().getClusterAsync(cluster) .whenComplete((clusterDataResult, ex) -> { if (ex != null) { + log.warn("[{}] Load cluster data failed: requested={}", clientAppId, cluster); clusterDataFuture.completeExceptionally(FutureUtil.unwrapCompletionException(ex)); return; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java index 62faa70bbcb76..008897136f8cf 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LeaderElectionServiceTest.java @@ -129,10 +129,10 @@ private void checkLookupException(String tenant, String namespace, PulsarClient .topic("persistent://" + tenant + "/" + namespace + "/1p") .create(); } catch (PulsarClientException t) { - Assert.assertTrue(t instanceof PulsarClientException.LookupException); + Assert.assertTrue(t instanceof PulsarClientException.BrokerMetadataException + || t instanceof PulsarClientException.LookupException); Assert.assertTrue( - t.getMessage().contains( - "java.lang.IllegalStateException: The leader election has not yet been completed!")); + t.getMessage().contains("The leader election has not yet been completed")); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index 4e24ffc98772d..96995499bf288 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -27,6 +27,7 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; import io.netty.handler.codec.http.HttpRequest; @@ -79,9 +80,13 @@ import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.namespace.OwnedBundle; +import org.apache.pulsar.broker.namespace.OwnershipCache; +import org.apache.pulsar.broker.namespace.ServiceUnitUtils; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.impl.BinaryProtoLookupService; +import org.apache.pulsar.client.impl.ClientCnx; import org.apache.pulsar.client.impl.LookupService; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.NamespaceBundle; @@ -95,6 +100,7 @@ import org.apache.pulsar.common.util.SecurityUtility; import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; +import org.apache.zookeeper.KeeperException; import org.asynchttpclient.AsyncCompletionHandler; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.AsyncHttpClientConfig; @@ -1116,4 +1122,101 @@ public String authenticate(AuthenticationDataSource authData) throws Authenticat return "invalid"; } } + + @Test + public void testLookupConnectionNotCloseIfGetUnloadingExOrMetadataEx() throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(tpName); + PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + Producer producer = pulsarClientImpl.newProducer(Schema.STRING).topic(tpName).create(); + Consumer consumer = pulsarClientImpl.newConsumer(Schema.STRING).topic(tpName) + .subscriptionName("s1").isAckReceiptEnabled(true).subscribe(); + LookupService lookupService = pulsarClientImpl.getLookup(); + assertTrue(lookupService instanceof BinaryProtoLookupService); + ClientCnx lookupConnection = pulsarClientImpl.getCnxPool().getConnection(lookupService.resolveHost()).join(); + + // Verify the socket will not be closed if the bundle is unloading. + BundleOfTopic bundleOfTopic = new BundleOfTopic(tpName); + bundleOfTopic.setBundleIsUnloading(); + try { + lookupService.getBroker(TopicName.get(tpName)).get(); + fail("It should failed due to the namespace bundle is unloading."); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("is being unloaded")); + } + // Do unload topic, trigger producer & consumer reconnection. + pulsar.getBrokerService().getTopic(tpName, false).join().get().close(true); + assertTrue(lookupConnection.ctx().channel().isActive()); + bundleOfTopic.setBundleIsNotUnloading(); + // Assert producer & consumer could reconnect successful. + producer.send("1"); + HashSet messagesReceived = new HashSet<>(); + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messagesReceived.add(msg.getValue()); + } + assertTrue(messagesReceived.contains("1")); + + // Verify the socket will not be closed if get a metadata ex. + bundleOfTopic.releaseBundleLockAndMakeAcquireFail(); + try { + lookupService.getBroker(TopicName.get(tpName)).get(); + fail("It should failed due to the acquire bundle lock fail."); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("OperationTimeout")); + } + // Do unload topic, trigger producer & consumer reconnection. + pulsar.getBrokerService().getTopic(tpName, false).join().get().close(true); + assertTrue(lookupConnection.ctx().channel().isActive()); + bundleOfTopic.makeAcquireBundleLockSuccess(); + // Assert producer could reconnect successful. + producer.send("2"); + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messagesReceived.add(msg.getValue()); + } + assertTrue(messagesReceived.contains("2")); + + // cleanup. + producer.close(); + consumer.close(); + admin.topics().delete(tpName); + } + + private class BundleOfTopic { + + private NamespaceBundle namespaceBundle; + private OwnershipCache ownershipCache; + private AsyncLoadingCache ownedBundlesCache; + + public BundleOfTopic(String tpName) { + namespaceBundle = pulsar.getNamespaceService().getBundle(TopicName.get(tpName)); + ownershipCache = pulsar.getNamespaceService().getOwnershipCache(); + ownedBundlesCache = WhiteboxImpl.getInternalState(ownershipCache, "ownedBundlesCache"); + } + + private void setBundleIsUnloading() { + ownedBundlesCache.get(namespaceBundle).join().setActive(false); + } + + private void setBundleIsNotUnloading() { + ownedBundlesCache.get(namespaceBundle).join().setActive(true); + } + + private void releaseBundleLockAndMakeAcquireFail() throws Exception { + ownedBundlesCache.synchronous().invalidateAll(); + mockZooKeeper.delete(ServiceUnitUtils.path(namespaceBundle), -1); + mockZooKeeper.setAlwaysFail(KeeperException.Code.OPERATIONTIMEOUT); + } + + private void makeAcquireBundleLockSuccess() throws Exception { + mockZooKeeper.unsetAlwaysFail(); + } + } } From eb4f5906f105bfc5006ed0fb6951ceafca53d6e1 Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Fri, 15 Sep 2023 21:12:39 +0800 Subject: [PATCH 303/494] [fix][broker] fix bug caused by optimistic locking (#18390) (cherry picked from commit 2aa8c3b1a284673b512db2b815514343ec3e7c19) --- .../collections/ConcurrentLongHashMap.java | 25 ++++--- .../ConcurrentLongLongPairHashMap.java | 39 ++++++----- .../collections/ConcurrentLongPairSet.java | 41 +++++++----- .../collections/ConcurrentOpenHashMap.java | 39 +++++++---- .../collections/ConcurrentOpenHashSet.java | 22 +++---- .../pulsar/common/util/FutureUtilTest.java | 1 - .../ConcurrentLongHashMapTest.java | 62 +++++++++++++++++- .../ConcurrentLongLongPairHashMapTest.java | 65 +++++++++++++++++++ .../ConcurrentLongPairSetTest.java | 64 ++++++++++++++++++ .../ConcurrentOpenHashMapTest.java | 64 ++++++++++++++++++ .../ConcurrentOpenHashSetTest.java | 64 ++++++++++++++++++ 11 files changed, 414 insertions(+), 72 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java index 31b4cb7cbf152..f5b47e7f1ab7a 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java @@ -306,16 +306,17 @@ private static final class Section extends StampedLock { } V get(long key, int keyHash) { - int bucket = keyHash; - long stamp = tryOptimisticRead(); boolean acquiredLock = false; + // add local variable here, so OutOfBound won't happen + long[] keys = this.keys; + V[] values = this.values; + // calculate table.length as capacity to avoid rehash changing capacity + int bucket = signSafeMod(keyHash, values.length); + try { while (true) { - int capacity = this.capacity; - bucket = signSafeMod(bucket, capacity); - // First try optimistic locking long storedKey = keys[bucket]; V storedValue = values[bucket]; @@ -333,16 +334,15 @@ V get(long key, int keyHash) { if (!acquiredLock) { stamp = readLock(); acquiredLock = true; + + // update local variable + keys = this.keys; + values = this.values; + bucket = signSafeMod(keyHash, values.length); storedKey = keys[bucket]; storedValue = values[bucket]; } - if (capacity != this.capacity) { - // There has been a rehashing. We need to restart the search - bucket = keyHash; - continue; - } - if (storedKey == key) { return storedValue != DeletedValue ? storedValue : null; } else if (storedValue == EmptyValue) { @@ -350,8 +350,7 @@ V get(long key, int keyHash) { return null; } } - - ++bucket; + bucket = (bucket + 1) & (values.length - 1); } } finally { if (acquiredLock) { diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java index c0ccad9b73d5b..c3babbb8d1103 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMap.java @@ -284,6 +284,9 @@ public Map asMap() { // A section is a portion of the hash map that is covered by a single @SuppressWarnings("serial") private static final class Section extends StampedLock { + // Each item take up 4 continuous array space. + private static final int ITEM_SIZE = 4; + // Keys and values are stored interleaved in the table array private volatile long[] table; @@ -306,7 +309,7 @@ private static final class Section extends StampedLock { float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); this.initCapacity = this.capacity; - this.table = new long[4 * this.capacity]; + this.table = new long[ITEM_SIZE * this.capacity]; this.size = 0; this.usedBuckets = 0; this.autoShrink = autoShrink; @@ -322,7 +325,10 @@ private static final class Section extends StampedLock { LongPair get(long key1, long key2, int keyHash) { long stamp = tryOptimisticRead(); boolean acquiredLock = false; - int bucket = signSafeMod(keyHash, capacity); + // add local variable here, so OutOfBound won't happen + long[] table = this.table; + // calculate table.length / 4 as capacity to avoid rehash changing capacity + int bucket = signSafeMod(keyHash, table.length / ITEM_SIZE); try { while (true) { @@ -345,8 +351,9 @@ LongPair get(long key1, long key2, int keyHash) { if (!acquiredLock) { stamp = readLock(); acquiredLock = true; - - bucket = signSafeMod(keyHash, capacity); + // update local variable + table = this.table; + bucket = signSafeMod(keyHash, table.length / ITEM_SIZE); storedKey1 = table[bucket]; storedKey2 = table[bucket + 1]; storedValue1 = table[bucket + 2]; @@ -361,7 +368,7 @@ LongPair get(long key1, long key2, int keyHash) { } } - bucket = (bucket + 4) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (acquiredLock) { @@ -413,7 +420,7 @@ boolean put(long key1, long key2, long value1, long value2, int keyHash, boolean } } - bucket = (bucket + 4) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (usedBuckets > resizeThresholdUp) { @@ -454,7 +461,7 @@ private boolean remove(long key1, long key2, long value1, long value2, int keyHa return false; } - bucket = (bucket + 4) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { @@ -480,7 +487,7 @@ private boolean remove(long key1, long key2, long value1, long value2, int keyHa } private void cleanBucket(int bucket) { - int nextInArray = (bucket + 4) & (table.length - 1); + int nextInArray = (bucket + ITEM_SIZE) & (table.length - 1); if (table[nextInArray] == EmptyKey) { table[bucket] = EmptyKey; table[bucket + 1] = EmptyKey; @@ -489,7 +496,7 @@ private void cleanBucket(int bucket) { --usedBuckets; // Cleanup all the buckets that were in `DeletedKey` state, so that we can reduce unnecessary expansions - bucket = (bucket - 4) & (table.length - 1); + bucket = (bucket - ITEM_SIZE) & (table.length - 1); while (table[bucket] == DeletedKey) { table[bucket] = EmptyKey; table[bucket + 1] = EmptyKey; @@ -497,7 +504,7 @@ private void cleanBucket(int bucket) { table[bucket + 3] = ValueNotFound; --usedBuckets; - bucket = (bucket - 4) & (table.length - 1); + bucket = (bucket - ITEM_SIZE) & (table.length - 1); } } else { table[bucket] = DeletedKey; @@ -540,7 +547,7 @@ public void forEach(BiConsumerLongPair processor) { } // Go through all the buckets for this section - for (int bucket = 0; bucket < table.length; bucket += 4) { + for (int bucket = 0; bucket < table.length; bucket += ITEM_SIZE) { long storedKey1 = table[bucket]; long storedKey2 = table[bucket + 1]; long storedValue1 = table[bucket + 2]; @@ -569,11 +576,11 @@ public void forEach(BiConsumerLongPair processor) { } private void rehash(int newCapacity) { - long[] newTable = new long[4 * newCapacity]; + long[] newTable = new long[ITEM_SIZE * newCapacity]; Arrays.fill(newTable, EmptyKey); // Re-hash table - for (int i = 0; i < table.length; i += 4) { + for (int i = 0; i < table.length; i += ITEM_SIZE) { long storedKey1 = table[i]; long storedKey2 = table[i + 1]; long storedValue1 = table[i + 2]; @@ -593,7 +600,7 @@ private void rehash(int newCapacity) { } private void shrinkToInitCapacity() { - long[] newTable = new long[4 * initCapacity]; + long[] newTable = new long[ITEM_SIZE * initCapacity]; Arrays.fill(newTable, EmptyKey); table = newTable; @@ -622,7 +629,7 @@ private static void insertKeyValueNoLock(long[] table, int capacity, long key1, return; } - bucket = (bucket + 4) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } } @@ -641,6 +648,8 @@ static final long hash(long key1, long key2) { } static final int signSafeMod(long n, int max) { + // as the ITEM_SIZE of Section is 4, so the index is the multiple of 4 + // that is to left shift 2 bits return (int) (n & (max - 1)) << 2; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java index 2a1090503857c..f0f04c4edf904 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java @@ -294,6 +294,9 @@ public Set items(int numberOfItems, LongPairFunction longPairConverter // A section is a portion of the hash map that is covered by a single @SuppressWarnings("serial") private static final class Section extends StampedLock { + // Each item take up 2 continuous array space. + private static final int ITEM_SIZE = 2; + // Keys and values are stored interleaved in the table array private volatile long[] table; @@ -315,7 +318,7 @@ private static final class Section extends StampedLock { float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); this.initCapacity = this.capacity; - this.table = new long[2 * this.capacity]; + this.table = new long[ITEM_SIZE * this.capacity]; this.size = 0; this.usedBuckets = 0; this.autoShrink = autoShrink; @@ -331,7 +334,11 @@ private static final class Section extends StampedLock { boolean contains(long item1, long item2, int hash) { long stamp = tryOptimisticRead(); boolean acquiredLock = false; - int bucket = signSafeMod(hash, capacity); + + // add local variable here, so OutOfBound won't happen + long[] table = this.table; + // calculate table.length / 2 as capacity to avoid rehash changing capacity + int bucket = signSafeMod(hash, table.length / ITEM_SIZE); try { while (true) { @@ -353,7 +360,9 @@ boolean contains(long item1, long item2, int hash) { stamp = readLock(); acquiredLock = true; - bucket = signSafeMod(hash, capacity); + // update local variable + table = this.table; + bucket = signSafeMod(hash, table.length / ITEM_SIZE); storedItem1 = table[bucket]; storedItem2 = table[bucket + 1]; } @@ -366,7 +375,7 @@ boolean contains(long item1, long item2, int hash) { } } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (acquiredLock) { @@ -410,7 +419,7 @@ boolean add(long item1, long item2, long hash) { } } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (usedBuckets > resizeThresholdUp) { @@ -445,7 +454,7 @@ private boolean remove(long item1, long item2, int hash) { return false; } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { tryShrinkThenUnlock(stamp); @@ -459,7 +468,7 @@ private int removeIf(LongPairPredicate filter) { // Go through all the buckets for this section long stamp = writeLock(); try { - for (int bucket = 0; bucket < table.length; bucket += 2) { + for (int bucket = 0; bucket < table.length; bucket += ITEM_SIZE) { long storedItem1 = table[bucket]; long storedItem2 = table[bucket + 1]; if (storedItem1 != DeletedItem && storedItem1 != EmptyItem) { @@ -498,7 +507,7 @@ private void tryShrinkThenUnlock(long stamp) { } private void cleanBucket(int bucket) { - int nextInArray = (bucket + 2) & (table.length - 1); + int nextInArray = (bucket + ITEM_SIZE) & (table.length - 1); if (table[nextInArray] == EmptyItem) { table[bucket] = EmptyItem; table[bucket + 1] = EmptyItem; @@ -506,13 +515,13 @@ private void cleanBucket(int bucket) { // Cleanup all the buckets that were in `DeletedItem` state, // so that we can reduce unnecessary expansions - int lastBucket = (bucket - 2) & (table.length - 1); + int lastBucket = (bucket - ITEM_SIZE) & (table.length - 1); while (table[lastBucket] == DeletedItem) { table[lastBucket] = EmptyItem; table[lastBucket + 1] = EmptyItem; --usedBuckets; - lastBucket = (lastBucket - 2) & (table.length - 1); + lastBucket = (lastBucket - ITEM_SIZE) & (table.length - 1); } } else { table[bucket] = DeletedItem; @@ -542,7 +551,7 @@ public void forEach(LongPairConsumer processor) { // Go through all the buckets for this section. We try to renew the stamp only after a validation // error, otherwise we keep going with the same. long stamp = 0; - for (int bucket = 0; bucket < table.length; bucket += 2) { + for (int bucket = 0; bucket < table.length; bucket += ITEM_SIZE) { if (stamp == 0) { stamp = tryOptimisticRead(); } @@ -572,11 +581,11 @@ public void forEach(LongPairConsumer processor) { private void rehash(int newCapacity) { // Expand the hashmap - long[] newTable = new long[2 * newCapacity]; + long[] newTable = new long[ITEM_SIZE * newCapacity]; Arrays.fill(newTable, EmptyItem); // Re-hash table - for (int i = 0; i < table.length; i += 2) { + for (int i = 0; i < table.length; i += ITEM_SIZE) { long storedItem1 = table[i]; long storedItem2 = table[i + 1]; if (storedItem1 != EmptyItem && storedItem1 != DeletedItem) { @@ -595,7 +604,7 @@ private void rehash(int newCapacity) { private void shrinkToInitCapacity() { // Expand the hashmap - long[] newTable = new long[2 * initCapacity]; + long[] newTable = new long[ITEM_SIZE * initCapacity]; Arrays.fill(newTable, EmptyItem); table = newTable; @@ -621,7 +630,7 @@ private static void insertKeyValueNoLock(long[] table, int capacity, long item1, return; } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } } @@ -640,6 +649,8 @@ static final long hash(long key1, long key2) { } static final int signSafeMod(long n, int max) { + // as the ITEM_SIZE of Section is 2, so the index is the multiple of 2 + // that is to left shift 1 bit return (int) (n & (max - 1)) << 1; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java index ea2e01768ac7e..1aa95d3090eb2 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMap.java @@ -296,6 +296,9 @@ public List values() { // A section is a portion of the hash map that is covered by a single @SuppressWarnings("serial") private static final class Section extends StampedLock { + // Each item take up 2 continuous array space. + private static final int ITEM_SIZE = 2; + // Keys and values are stored interleaved in the table array private volatile Object[] table; @@ -317,7 +320,7 @@ private static final class Section extends StampedLock { float expandFactor, float shrinkFactor) { this.capacity = alignToPowerOfTwo(capacity); this.initCapacity = this.capacity; - this.table = new Object[2 * this.capacity]; + this.table = new Object[ITEM_SIZE * this.capacity]; this.size = 0; this.usedBuckets = 0; this.autoShrink = autoShrink; @@ -332,7 +335,11 @@ private static final class Section extends StampedLock { V get(K key, int keyHash) { long stamp = tryOptimisticRead(); boolean acquiredLock = false; - int bucket = signSafeMod(keyHash, capacity); + + // add local variable here, so OutOfBound won't happen + Object[] table = this.table; + // calculate table.length / 2 as capacity to avoid rehash changing capacity + int bucket = signSafeMod(keyHash, table.length / ITEM_SIZE); try { while (true) { @@ -354,7 +361,9 @@ V get(K key, int keyHash) { stamp = readLock(); acquiredLock = true; - bucket = signSafeMod(keyHash, capacity); + // update local variable + table = this.table; + bucket = signSafeMod(keyHash, table.length / ITEM_SIZE); storedKey = (K) table[bucket]; storedValue = (V) table[bucket + 1]; } @@ -367,7 +376,7 @@ V get(K key, int keyHash) { } } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (acquiredLock) { @@ -420,7 +429,7 @@ V put(K key, V value, int keyHash, boolean onlyIfAbsent, Function valuePro } } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { if (usedBuckets > resizeThresholdUp) { @@ -449,7 +458,7 @@ private V remove(K key, Object value, int keyHash) { if (value == null || value.equals(storedValue)) { SIZE_UPDATER.decrementAndGet(this); - int nextInArray = (bucket + 2) & (table.length - 1); + int nextInArray = (bucket + ITEM_SIZE) & (table.length - 1); if (table[nextInArray] == EmptyKey) { table[bucket] = EmptyKey; table[bucket + 1] = null; @@ -457,13 +466,13 @@ private V remove(K key, Object value, int keyHash) { // Cleanup all the buckets that were in `DeletedKey` state, // so that we can reduce unnecessary expansions - int lastBucket = (bucket - 2) & (table.length - 1); + int lastBucket = (bucket - ITEM_SIZE) & (table.length - 1); while (table[lastBucket] == DeletedKey) { table[lastBucket] = EmptyKey; table[lastBucket + 1] = null; --usedBuckets; - lastBucket = (lastBucket - 2) & (table.length - 1); + lastBucket = (lastBucket - ITEM_SIZE) & (table.length - 1); } } else { table[bucket] = DeletedKey; @@ -479,7 +488,7 @@ private V remove(K key, Object value, int keyHash) { return null; } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } finally { @@ -528,7 +537,7 @@ public void forEach(BiConsumer processor) { // Go through all the buckets for this section. We try to renew the stamp only after a validation // error, otherwise we keep going with the same. long stamp = 0; - for (int bucket = 0; bucket < table.length; bucket += 2) { + for (int bucket = 0; bucket < table.length; bucket += ITEM_SIZE) { if (stamp == 0) { stamp = tryOptimisticRead(); } @@ -558,10 +567,10 @@ public void forEach(BiConsumer processor) { private void rehash(int newCapacity) { // Expand the hashmap - Object[] newTable = new Object[2 * newCapacity]; + Object[] newTable = new Object[ITEM_SIZE * newCapacity]; // Re-hash table - for (int i = 0; i < table.length; i += 2) { + for (int i = 0; i < table.length; i += ITEM_SIZE) { K storedKey = (K) table[i]; V storedValue = (V) table[i + 1]; if (storedKey != EmptyKey && storedKey != DeletedKey) { @@ -577,7 +586,7 @@ private void rehash(int newCapacity) { } private void shrinkToInitCapacity() { - Object[] newTable = new Object[2 * initCapacity]; + Object[] newTable = new Object[ITEM_SIZE * initCapacity]; table = newTable; size = 0; @@ -602,7 +611,7 @@ private static void insertKeyValueNoLock(Object[] table, int capacity, K return; } - bucket = (bucket + 2) & (table.length - 1); + bucket = (bucket + ITEM_SIZE) & (table.length - 1); } } } @@ -618,6 +627,8 @@ static final long hash(K key) { } static final int signSafeMod(long n, int max) { + // as the ITEM_SIZE of Section is 2, so the index is the multiple of 2 + // that is to left shift 1 bit return (int) (n & (max - 1)) << 1; } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java index cc8bc07b43095..162cbed5a5ddc 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSet.java @@ -294,16 +294,16 @@ private static final class Section extends StampedLock { } boolean contains(V value, int keyHash) { - int bucket = keyHash; - long stamp = tryOptimisticRead(); boolean acquiredLock = false; + // add local variable here, so OutOfBound won't happen + V[] values = this.values; + // calculate table.length as capacity to avoid rehash changing capacity + int bucket = signSafeMod(keyHash, values.length); + try { while (true) { - int capacity = this.capacity; - bucket = signSafeMod(bucket, capacity); - // First try optimistic locking V storedValue = values[bucket]; @@ -321,15 +321,12 @@ boolean contains(V value, int keyHash) { stamp = readLock(); acquiredLock = true; + // update local variable + values = this.values; + bucket = signSafeMod(keyHash, values.length); storedValue = values[bucket]; } - if (capacity != this.capacity) { - // There has been a rehashing. We need to restart the search - bucket = keyHash; - continue; - } - if (value.equals(storedValue)) { return true; } else if (storedValue == EmptyValue) { @@ -337,8 +334,7 @@ boolean contains(V value, int keyHash) { return false; } } - - ++bucket; + bucket = (bucket + 1) & (values.length - 1); } } finally { if (acquiredLock) { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java index 6df4494edf886..7d44c187d7355 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FutureUtilTest.java @@ -181,7 +181,6 @@ public void testWaitForAny() { } } - @Test public void testSequencer() { int concurrentNum = 1000; final ScheduledExecutorService executor = Executors.newScheduledThreadPool(concurrentNum); diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java index a317fa63c0986..e1f947ad8c4f6 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMapTest.java @@ -38,6 +38,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongFunction; import lombok.Cleanup; @@ -213,6 +214,66 @@ public void testExpandShrinkAndClear() { assertTrue(map.capacity() == initCapacity); } + @Test + public void testConcurrentExpandAndShrinkAndGet() throws Throwable { + ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertEquals(map.capacity(), 4); + + ExecutorService executor = Executors.newCachedThreadPool(); + final int readThreads = 16; + final int writeThreads = 1; + final int n = 1_000; + CyclicBarrier barrier = new CyclicBarrier(writeThreads + readThreads); + Future future = null; + AtomicReference ex = new AtomicReference<>(); + + for (int i = 0; i < readThreads; i++) { + executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + try { + map.get(1); + } catch (Exception e) { + ex.set(e); + } + }); + } + + assertNull(map.put(1,"v1")); + future = executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + for (int i = 0; i < n; i++) { + // expand hashmap + assertNull(map.put(2, "v2")); + assertNull(map.put(3, "v3")); + assertEquals(map.capacity(), 8); + + // shrink hashmap + assertTrue(map.remove(2, "v2")); + assertTrue(map.remove(3, "v3")); + assertEquals(map.capacity(), 4); + } + }); + + future.get(); + assertTrue(ex.get() == null); + // shut down pool + executor.shutdown(); + } + @Test public void testRemove() { ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() @@ -361,7 +422,6 @@ public void concurrentInsertionsAndReads() throws Throwable { assertEquals(map.size(), N * nThreads); } - @Test public void stressConcurrentInsertionsAndReads() throws Throwable { ConcurrentLongHashMap map = ConcurrentLongHashMap.newBuilder() .expectedItems(4) diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java index 8e74d285ffb9b..0de3fdb5c84bf 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongLongPairHashMapTest.java @@ -31,9 +31,12 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; + import org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair; import org.testng.annotations.Test; @@ -173,6 +176,68 @@ public void testExpandAndShrink() { assertEquals(map.capacity(), 8); } + @Test + public void testConcurrentExpandAndShrinkAndGet() throws Throwable { + ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertEquals(map.capacity(), 4); + + ExecutorService executor = Executors.newCachedThreadPool(); + final int readThreads = 16; + final int writeThreads = 1; + final int n = 1_000; + CyclicBarrier barrier = new CyclicBarrier(writeThreads + readThreads); + Future future = null; + AtomicReference ex = new AtomicReference<>(); + + for (int i = 0; i < readThreads; i++) { + executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + while (true) { + try { + map.get(1, 1); + } catch (Exception e) { + ex.set(e); + } + } + }); + } + + assertTrue(map.put(1, 1, 11, 11)); + future = executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + for (int i = 0; i < n; i++) { + // expand hashmap + assertTrue(map.put(2, 2, 22, 22)); + assertTrue(map.put(3, 3, 33, 33)); + assertEquals(map.capacity(), 8); + + // shrink hashmap + assertTrue(map.remove(2, 2, 22, 22)); + assertTrue(map.remove(3, 3, 33, 33)); + assertEquals(map.capacity(), 4); + } + }); + + future.get(); + assertTrue(ex.get() == null); + // shut down pool + executor.shutdown(); + } + @Test public void testExpandShrinkAndClear() { ConcurrentLongLongPairHashMap map = ConcurrentLongLongPairHashMap.newBuilder() diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java index 7e947ae6e6aa3..bce2b8993835f 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSetTest.java @@ -30,9 +30,11 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.apache.pulsar.common.util.collections.ConcurrentLongPairSet.LongPair; @@ -211,6 +213,68 @@ public void testExpandShrinkAndClear() { assertTrue(map.capacity() == initCapacity); } + @Test + public void testConcurrentExpandAndShrinkAndGet() throws Throwable { + ConcurrentLongPairSet set = ConcurrentLongPairSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertEquals(set.capacity(), 4); + + ExecutorService executor = Executors.newCachedThreadPool(); + final int readThreads = 16; + final int writeThreads = 1; + final int n = 1_000; + CyclicBarrier barrier = new CyclicBarrier(writeThreads + readThreads); + Future future = null; + AtomicReference ex = new AtomicReference<>(); + + for (int i = 0; i < readThreads; i++) { + executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + while (true) { + try { + set.contains(1, 1); + } catch (Exception e) { + ex.set(e); + } + } + }); + } + + assertTrue(set.add(1, 1)); + future = executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + for (int i = 0; i < n; i++) { + // expand hashmap + assertTrue(set.add(2, 2)); + assertTrue(set.add(3, 3)); + assertEquals(set.capacity(), 8); + + // shrink hashmap + assertTrue(set.remove(2, 2)); + assertTrue(set.remove(3, 3)); + assertEquals(set.capacity(), 4); + } + }); + + future.get(); + assertTrue(ex.get() == null); + // shut down pool + executor.shutdown(); + } + @Test public void testRemove() { diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java index 198a3f4c5c38b..410d490b98faa 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashMapTest.java @@ -32,11 +32,13 @@ import java.util.List; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import lombok.Cleanup; @@ -215,6 +217,68 @@ public void testExpandShrinkAndClear() { assertTrue(map.capacity() == initCapacity); } + @Test + public void testConcurrentExpandAndShrinkAndGet() throws Throwable { + ConcurrentOpenHashMap map = ConcurrentOpenHashMap.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertEquals(map.capacity(), 4); + + ExecutorService executor = Executors.newCachedThreadPool(); + final int readThreads = 16; + final int writeThreads = 1; + final int n = 1_000; + CyclicBarrier barrier = new CyclicBarrier(writeThreads + readThreads); + Future future = null; + AtomicReference ex = new AtomicReference<>(); + + for (int i = 0; i < readThreads; i++) { + executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + while (true) { + try { + map.get("k2"); + } catch (Exception e) { + ex.set(e); + } + } + }); + } + + assertNull(map.put("k1","v1")); + future = executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + for (int i = 0; i < n; i++) { + // expand hashmap + assertNull(map.put("k2", "v2")); + assertNull(map.put("k3", "v3")); + assertEquals(map.capacity(), 8); + + // shrink hashmap + assertTrue(map.remove("k2", "v2")); + assertTrue(map.remove("k3", "v3")); + assertEquals(map.capacity(), 4); + } + }); + + future.get(); + assertTrue(ex.get() == null); + // shut down pool + executor.shutdown(); + } + @Test public void testRemove() { ConcurrentOpenHashMap map = diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java index 27c18abb8b347..6a40095ab0647 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/collections/ConcurrentOpenHashSetTest.java @@ -28,9 +28,11 @@ import java.util.Collections; import java.util.List; import java.util.Random; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.testng.annotations.Test; @@ -186,6 +188,68 @@ public void testExpandShrinkAndClear() { assertTrue(map.capacity() == initCapacity); } + @Test + public void testConcurrentExpandAndShrinkAndGet() throws Throwable { + ConcurrentOpenHashSet set = ConcurrentOpenHashSet.newBuilder() + .expectedItems(2) + .concurrencyLevel(1) + .autoShrink(true) + .mapIdleFactor(0.25f) + .build(); + assertEquals(set.capacity(), 4); + + ExecutorService executor = Executors.newCachedThreadPool(); + final int readThreads = 16; + final int writeThreads = 1; + final int n = 1_000; + CyclicBarrier barrier = new CyclicBarrier(writeThreads + readThreads); + Future future = null; + AtomicReference ex = new AtomicReference<>(); + + for (int i = 0; i < readThreads; i++) { + executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + while (true) { + try { + set.contains("k2"); + } catch (Exception e) { + ex.set(e); + } + } + }); + } + + assertTrue(set.add("k1")); + future = executor.submit(() -> { + try { + barrier.await(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + for (int i = 0; i < n; i++) { + // expand hashmap + assertTrue(set.add("k2")); + assertTrue(set.add("k3")); + assertEquals(set.capacity(), 8); + + // shrink hashmap + assertTrue(set.remove("k2")); + assertTrue(set.remove("k3")); + assertEquals(set.capacity(), 4); + } + }); + + future.get(); + assertTrue(ex.get() == null); + // shut down pool + executor.shutdown(); + } + @Test public void testRemove() { ConcurrentOpenHashSet set = From 3129621f894f20085b9ddec02892425ddf7e0c14 Mon Sep 17 00:00:00 2001 From: ken <1647023764@qq.com> Date: Sat, 7 Oct 2023 20:24:45 +0800 Subject: [PATCH 304/494] [fix][broker] rackaware policy is ineffective when delete zk rack info after bkclient initialize (#20944) (cherry picked from commit d9ebaf5bf6fda44d21ac24cec7dbe208b59dc597) --- .../BookieRackAffinityMapping.java | 4 ++-- .../BookieRackAffinityMappingTest.java | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java index e9e350800b44e..d54ef2a5f4cef 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMapping.java @@ -121,8 +121,6 @@ public synchronized void setConf(Configuration conf) { store.registerListener(this::handleUpdates); racksWithHost = bookieMappingCache.get(BOOKIE_INFO_ROOT_PATH).get() .orElseGet(BookiesRackConfiguration::new); - updateRacksWithHost(racksWithHost); - watchAvailableBookies(); for (Map bookieMapping : racksWithHost.values()) { for (String address : bookieMapping.keySet()) { bookieAddressListLastTime.add(BookieId.parse(address)); @@ -132,6 +130,8 @@ public synchronized void setConf(Configuration conf) { bookieAddressListLastTime); } } + updateRacksWithHost(racksWithHost); + watchAvailableBookies(); } catch (InterruptedException | ExecutionException | MetadataException e) { throw new RuntimeException(METADATA_STORE_INSTANCE + " failed to init BookieId list"); } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java index d7be7dabd0db1..d7df5afb4bebe 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/bookie/rackawareness/BookieRackAffinityMappingTest.java @@ -254,6 +254,7 @@ public void testWithPulsarRegistrationClient() throws Exception { bkClientConf.getTimeoutTimerNumTicks()); RackawareEnsemblePlacementPolicy repp = new RackawareEnsemblePlacementPolicy(); + mapping.registerRackChangeListener(repp); Class clazz1 = Class.forName("org.apache.bookkeeper.client.TopologyAwareEnsemblePlacementPolicy"); Field field1 = clazz1.getDeclaredField("knownBookies"); field1.setAccessible(true); @@ -323,6 +324,22 @@ public void testWithPulsarRegistrationClient() throws Exception { assertEquals(knownBookies.get(BOOKIE2.toBookieId()).getNetworkLocation(), "/rack1"); assertEquals(knownBookies.get(BOOKIE3.toBookieId()).getNetworkLocation(), "/default-rack"); + //remove bookie2 rack, the bookie2 rack should be /default-rack + data = "{\"group1\": {\"" + BOOKIE1 + + "\": {\"rack\": \"/rack0\", \"hostname\": \"bookie1.example.com\"}}}"; + store.put(BookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, data.getBytes(), Optional.empty()).join(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> ((BookiesRackConfiguration)field.get(mapping)).get("group1").size() == 1); + + racks = mapping + .resolve(Lists.newArrayList(BOOKIE1.getHostName(), BOOKIE2.getHostName(), BOOKIE3.getHostName())) + .stream().filter(Objects::nonNull).toList(); + assertEquals(racks.size(), 1); + assertEquals(racks.get(0), "/rack0"); + assertEquals(knownBookies.size(), 3); + assertEquals(knownBookies.get(BOOKIE1.toBookieId()).getNetworkLocation(), "/rack0"); + assertEquals(knownBookies.get(BOOKIE2.toBookieId()).getNetworkLocation(), "/default-rack"); + assertEquals(knownBookies.get(BOOKIE3.toBookieId()).getNetworkLocation(), "/default-rack"); + timer.stop(); } } From a0e2104642c5081986dbe7847c71997b06030eaa Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 26 Sep 2023 10:33:35 +0800 Subject: [PATCH 305/494] [fix][broker] Fix inconsistent topic policy (#21231) (cherry picked from commit afc924405829f23962e04a717571f33930c165d6) --- .../ProxySaslAuthenticationTest.java | 13 +- .../authentication/SaslAuthenticateTest.java | 11 +- .../pulsar/broker/service/BrokerService.java | 295 +++++++++--------- .../SystemTopicBasedTopicPoliciesService.java | 100 ++++-- .../broker/service/TopicPoliciesService.java | 41 +++ .../broker/admin/PersistentTopicsTest.java | 1 + .../broker/admin/TopicPoliciesTest.java | 2 +- .../TopicPoliciesWithBrokerRestartTest.java | 104 ++++++ .../pulsar/broker/admin/TopicsAuthTest.java | 2 + .../pulsar/broker/auth/AuthLogsTest.java | 2 + .../broker/auth/MockAuthentication.java | 6 +- .../auth/MockedPulsarServiceBaseTest.java | 19 ++ .../service/BrokerBookieIsolationTest.java | 3 +- ...temTopicBasedTopicPoliciesServiceTest.java | 6 +- .../persistent/PersistentTopicTest.java | 3 +- ...enticationTlsHostnameVerificationTest.java | 3 + .../AuthorizationProducerConsumerTest.java | 5 + .../client/api/MutualAuthenticationTest.java | 2 +- ...okenAuthenticatedProducerConsumerTest.java | 3 + ...uth2AuthenticatedProducerConsumerTest.java | 14 +- ...reTlsProducerConsumerTestWithAuthTest.java | 22 ++ .../PatternTopicsConsumerImplAuthTest.java | 1 + .../standalone_no_client_auth.conf | 3 +- .../proxy/server/ProxyAuthenticationTest.java | 2 +- .../server/ProxyForwardAuthDataTest.java | 2 +- .../server/ProxyRolesEnforcementTest.java | 2 +- .../server/ProxyWithAuthorizationNegTest.java | 2 + .../pulsar/sql/presto/TestPulsarAuth.java | 4 + .../ExtensibleLoadManagerTest.java | 1 + .../integration/presto/TestPulsarSQLAuth.java | 1 + 30 files changed, 478 insertions(+), 197 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java index 261efe680f862..f0e45aa734afb 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java @@ -49,6 +49,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.auth.AuthenticationSasl; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.proxy.server.ProxyConfiguration; import org.apache.pulsar.proxy.server.ProxyService; import org.slf4j.Logger; @@ -193,15 +194,17 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setClusterName("test"); conf.setSuperUserRoles(ImmutableSet.of("client/" + localHostname + "@" + kdc.getRealm())); - - super.init(); - - lookupUrl = new URI(pulsar.getBrokerServiceUrl()); - // set admin auth, to verify admin web resources Map clientSaslConfig = new HashMap<>(); clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); clientSaslConfig.put("serverType", "broker"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(clientSaslConfig)); + + super.init(); + + lookupUrl = new URI(pulsar.getBrokerServiceUrl()); log.info("set client jaas section name: PulsarClient"); admin = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl.toString()) diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java index 8a0d0392d1333..76c6f023d9a36 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java @@ -53,6 +53,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationSasl; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.sasl.SaslConstants; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; @@ -180,7 +181,12 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setClusterName("test"); conf.setSuperUserRoles(ImmutableSet.of("client" + "@" + kdc.getRealm())); - + Map clientSaslConfig = new HashMap<>(); + clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); + clientSaslConfig.put("serverType", "broker"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(clientSaslConfig)); super.init(); lookupUrl = new URI(pulsar.getWebServiceAddress()); @@ -191,9 +197,6 @@ protected void setup() throws Exception { .authentication(authSasl)); // set admin auth, to verify admin web resources - Map clientSaslConfig = new HashMap<>(); - clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); - clientSaslConfig.put("serverType", "broker"); log.info("set client jaas section name: PulsarClient"); admin = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl.toString()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index d6b17f4faa4da..7c7dec864e2e8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1769,165 +1769,172 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { }); } - public CompletableFuture getManagedLedgerConfig(TopicName topicName) { + public CompletableFuture getManagedLedgerConfig(@Nonnull TopicName topicName) { + requireNonNull(topicName); NamespaceName namespace = topicName.getNamespaceObject(); ServiceConfiguration serviceConfig = pulsar.getConfiguration(); NamespaceResources nsr = pulsar.getPulsarResources().getNamespaceResources(); LocalPoliciesResources lpr = pulsar.getPulsarResources().getLocalPolicies(); - return nsr.getPoliciesAsync(namespace) - .thenCombine(lpr.getLocalPoliciesAsync(namespace), (policies, localPolicies) -> { - PersistencePolicies persistencePolicies = null; - RetentionPolicies retentionPolicies = null; - OffloadPoliciesImpl topicLevelOffloadPolicies = null; - - if (pulsar.getConfig().isTopicLevelPoliciesEnabled() - && !NamespaceService.isSystemServiceNamespace(namespace.toString())) { - final TopicPolicies topicPolicies = pulsar.getTopicPoliciesService() - .getTopicPoliciesIfExists(topicName); - if (topicPolicies != null) { - persistencePolicies = topicPolicies.getPersistence(); - retentionPolicies = topicPolicies.getRetentionPolicies(); - topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies(); - } - } - - if (persistencePolicies == null) { - persistencePolicies = policies.map(p -> p.persistence).orElseGet( - () -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), - serviceConfig.getManagedLedgerDefaultWriteQuorum(), - serviceConfig.getManagedLedgerDefaultAckQuorum(), - serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit())); - } + final CompletableFuture> topicPoliciesFuture; + if (pulsar.getConfig().isTopicLevelPoliciesEnabled() + && !NamespaceService.isSystemServiceNamespace(namespace.toString()) + && !SystemTopicNames.isTopicPoliciesSystemTopic(topicName.toString())) { + topicPoliciesFuture = pulsar.getTopicPoliciesService().getTopicPoliciesAsync(topicName); + } else { + topicPoliciesFuture = CompletableFuture.completedFuture(Optional.empty()); + } + return topicPoliciesFuture.thenCompose(topicPoliciesOptional -> { + final CompletableFuture> nsPolicies = nsr.getPoliciesAsync(namespace); + final CompletableFuture> lcPolicies = lpr.getLocalPoliciesAsync(namespace); + return nsPolicies.thenCombine(lcPolicies, (policies, localPolicies) -> { + PersistencePolicies persistencePolicies = null; + RetentionPolicies retentionPolicies = null; + OffloadPoliciesImpl topicLevelOffloadPolicies = null; + if (topicPoliciesOptional.isPresent()) { + final TopicPolicies topicPolicies = topicPoliciesOptional.get(); + persistencePolicies = topicPolicies.getPersistence(); + retentionPolicies = topicPolicies.getRetentionPolicies(); + topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies(); + } - if (retentionPolicies == null) { - retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( - () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), - serviceConfig.getDefaultRetentionSizeInMB()) - ); - } + if (persistencePolicies == null) { + persistencePolicies = policies.map(p -> p.persistence).orElseGet( + () -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), + serviceConfig.getManagedLedgerDefaultWriteQuorum(), + serviceConfig.getManagedLedgerDefaultAckQuorum(), + serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit())); + } - ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble()); - managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum()); - managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum()); + if (retentionPolicies == null) { + retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( + () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), + serviceConfig.getDefaultRetentionSizeInMB()) + ); + } - if (serviceConfig.isStrictBookieAffinityEnabled()) { + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble()); + managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum()); + managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum()); + + if (serviceConfig.isStrictBookieAffinityEnabled()) { + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( + IsolatedBookieEnsemblePlacementPolicy.class); + if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } else if (isSystemTopic(topicName)) { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); + properties.put(IsolatedBookieEnsemblePlacementPolicy + .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } else { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } + } else { + if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( IsolatedBookieEnsemblePlacementPolicy.class); - if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } else if (isSystemTopic(topicName)) { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); - properties.put(IsolatedBookieEnsemblePlacementPolicy - .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } else { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } - } else { - if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( - IsolatedBookieEnsemblePlacementPolicy.class); - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); } + } - managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate()); - managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType()); - managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword()); - - managedLedgerConfig - .setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist()); - managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( - serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); - managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( - serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); - managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); - managedLedgerConfig - .setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), - TimeUnit.MINUTES); - managedLedgerConfig - .setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), - TimeUnit.MINUTES); - managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes()); - - managedLedgerConfig.setMetadataOperationsTimeoutSeconds( - serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds()); - managedLedgerConfig - .setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds()); - managedLedgerConfig - .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); - managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( - serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); - managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); - managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); - managedLedgerConfig - .setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger()); - - managedLedgerConfig - .setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds()); - managedLedgerConfig - .setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); - managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); - managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData()); - managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery()); - managedLedgerConfig.setInactiveLedgerRollOverTime( - serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS); - managedLedgerConfig.setCacheEvictionByMarkDeletedPosition( - serviceConfig.isCacheEvictionByMarkDeletedPosition()); - managedLedgerConfig.setMinimumBacklogCursorsForCaching( - serviceConfig.getManagedLedgerMinimumBacklogCursorsForCaching()); - managedLedgerConfig.setMinimumBacklogEntriesForCaching( - serviceConfig.getManagedLedgerMinimumBacklogEntriesForCaching()); - managedLedgerConfig.setMaxBacklogBetweenCursorsForCaching( - serviceConfig.getManagedLedgerMaxBacklogBetweenCursorsForCaching()); - - OffloadPoliciesImpl nsLevelOffloadPolicies = - (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null); - OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration( - topicLevelOffloadPolicies, - OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), - getPulsar().getConfig().getProperties()); - if (NamespaceService.isSystemServiceNamespace(namespace.toString())) { - managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE); - } else { - if (topicLevelOffloadPolicies != null) { - try { - LedgerOffloader topicLevelLedgerOffLoader = - pulsar().createManagedLedgerOffloader(offloadPolicies); - managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); - } catch (PulsarServerException e) { - throw new RuntimeException(e); - } - } else { - //If the topic level policy is null, use the namespace level - managedLedgerConfig - .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); + managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate()); + managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType()); + managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword()); + + managedLedgerConfig + .setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist()); + managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( + serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); + managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( + serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); + managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); + managedLedgerConfig + .setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), + TimeUnit.MINUTES); + managedLedgerConfig + .setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), + TimeUnit.MINUTES); + managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes()); + + managedLedgerConfig.setMetadataOperationsTimeoutSeconds( + serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds()); + managedLedgerConfig + .setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds()); + managedLedgerConfig + .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); + managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); + managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( + serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); + managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); + managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); + managedLedgerConfig + .setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger()); + + managedLedgerConfig + .setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds()); + managedLedgerConfig + .setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); + managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); + managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData()); + managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery()); + managedLedgerConfig.setInactiveLedgerRollOverTime( + serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS); + managedLedgerConfig.setCacheEvictionByMarkDeletedPosition( + serviceConfig.isCacheEvictionByMarkDeletedPosition()); + managedLedgerConfig.setMinimumBacklogCursorsForCaching( + serviceConfig.getManagedLedgerMinimumBacklogCursorsForCaching()); + managedLedgerConfig.setMinimumBacklogEntriesForCaching( + serviceConfig.getManagedLedgerMinimumBacklogEntriesForCaching()); + managedLedgerConfig.setMaxBacklogBetweenCursorsForCaching( + serviceConfig.getManagedLedgerMaxBacklogBetweenCursorsForCaching()); + + OffloadPoliciesImpl nsLevelOffloadPolicies = + (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null); + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration( + topicLevelOffloadPolicies, + OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), + getPulsar().getConfig().getProperties()); + if (NamespaceService.isSystemServiceNamespace(namespace.toString())) { + managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE); + } else { + if (topicLevelOffloadPolicies != null) { + try { + LedgerOffloader topicLevelLedgerOffLoader = + pulsar().createManagedLedgerOffloader(offloadPolicies); + managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); + } catch (PulsarServerException e) { + throw new RuntimeException(e); } + } else { + //If the topic level policy is null, use the namespace level + managedLedgerConfig + .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); } + } - managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( - serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled()); - managedLedgerConfig.setNewEntriesCheckDelayInMillis( - serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis()); - return managedLedgerConfig; - }); + managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( + serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled()); + managedLedgerConfig.setNewEntriesCheckDelayInMillis( + serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis()); + return managedLedgerConfig; + }); + }); } private void addTopicToStatsMaps(TopicName topicName, Topic topic) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 09f8de818db0a..ed76d37ae2536 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -18,12 +18,14 @@ */ package org.apache.pulsar.broker.service; +import static java.util.Objects.requireNonNull; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -54,6 +56,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.FutureUtil; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,8 +81,8 @@ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesServic private final Map>> readerCaches = new ConcurrentHashMap<>(); - @VisibleForTesting - final Map policyCacheInitMap = new ConcurrentHashMap<>(); + + final Map> policyCacheInitMap = new ConcurrentHashMap<>(); @VisibleForTesting final Map>> listeners = new ConcurrentHashMap<>(); @@ -219,12 +222,12 @@ public TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) throws TopicPoliciesCacheNotInitException { if (!policyCacheInitMap.containsKey(topicName.getNamespaceObject())) { NamespaceName namespace = topicName.getNamespaceObject(); - prepareInitPoliciesCache(namespace, new CompletableFuture<>()); + prepareInitPoliciesCacheAsync(namespace); } MutablePair result = new MutablePair<>(); policyCacheInitMap.compute(topicName.getNamespaceObject(), (k, initialized) -> { - if (initialized == null || !initialized) { + if (initialized == null || !initialized.isDone()) { result.setLeft(new TopicPoliciesCacheNotInitException()); } else { TopicPolicies topicPolicies = @@ -242,6 +245,34 @@ public TopicPolicies getTopicPolicies(TopicName topicName, } } + @NotNull + @Override + public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName, + boolean isGlobal) { + requireNonNull(topicName); + final CompletableFuture preparedFuture = prepareInitPoliciesCacheAsync(topicName.getNamespaceObject()); + return preparedFuture.thenApply(__ -> { + final TopicPolicies candidatePolicies = isGlobal + ? globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName())) + : policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); + return Optional.ofNullable(candidatePolicies); + }); + } + + @NotNull + @Override + public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName) { + requireNonNull(topicName); + final CompletableFuture preparedFuture = prepareInitPoliciesCacheAsync(topicName.getNamespaceObject()); + return preparedFuture.thenApply(__ -> { + final TopicPolicies localPolicies = policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); + if (localPolicies != null) { + return Optional.of(localPolicies); + } + return Optional.ofNullable(globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName()))); + }); + } + @Override public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { return policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); @@ -265,39 +296,48 @@ public CompletableFuture getTopicPoliciesBypassCacheAsync(TopicNa @Override public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { - CompletableFuture result = new CompletableFuture<>(); NamespaceName namespace = namespaceBundle.getNamespaceObject(); if (NamespaceService.isHeartbeatNamespace(namespace)) { - result.complete(null); - return result; + return CompletableFuture.completedFuture(null); } synchronized (this) { if (readerCaches.get(namespace) != null) { ownedBundlesCountPerNamespace.get(namespace).incrementAndGet(); - result.complete(null); + return CompletableFuture.completedFuture(null); } else { - prepareInitPoliciesCache(namespace, result); + return prepareInitPoliciesCacheAsync(namespace); } } - return result; } - private void prepareInitPoliciesCache(@Nonnull NamespaceName namespace, CompletableFuture result) { - if (policyCacheInitMap.putIfAbsent(namespace, false) == null) { - CompletableFuture> readerCompletableFuture = + private @Nonnull CompletableFuture prepareInitPoliciesCacheAsync(@Nonnull NamespaceName namespace) { + requireNonNull(namespace); + return policyCacheInitMap.computeIfAbsent(namespace, (k) -> { + final CompletableFuture> readerCompletableFuture = createSystemTopicClientWithRetry(namespace); readerCaches.put(namespace, readerCompletableFuture); ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); - readerCompletableFuture.thenAccept(reader -> { - initPolicesCache(reader, result); - result.thenRun(() -> readMorePolicies(reader)); - }).exceptionally(ex -> { - log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); - cleanCacheAndCloseReader(namespace, false); - result.completeExceptionally(ex); + final CompletableFuture initFuture = readerCompletableFuture + .thenCompose(reader -> { + final CompletableFuture stageFuture = new CompletableFuture<>(); + initPolicesCache(reader, stageFuture); + return stageFuture + // Read policies in background + .thenAccept(__ -> readMorePoliciesAsync(reader)); + }); + initFuture.exceptionally(ex -> { + try { + log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); + cleanCacheAndCloseReader(namespace, false); + } catch (Throwable cleanupEx) { + // Adding this catch to avoid break callback chain + log.error("[{}] Failed to cleanup reader on __change_events topic", namespace, cleanupEx); + } return null; }); - } + // let caller know we've got an exception. + return initFuture; + }); } protected CompletableFuture> createSystemTopicClientWithRetry( @@ -381,8 +421,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp if (log.isDebugEnabled()) { log.debug("[{}] Reach the end of the system topic.", reader.getSystemTopic().getTopicName()); } - policyCacheInitMap.computeIfPresent( - reader.getSystemTopic().getTopicName().getNamespaceObject(), (k, v) -> true); + // replay policy message policiesCache.forEach(((topicName, topicPolicies) -> { if (listeners.get(topicName) != null) { @@ -395,6 +434,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp } } })); + future.complete(null); } }); @@ -420,7 +460,13 @@ private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean }); } - private void readMorePolicies(SystemTopicClient.Reader reader) { + /** + * This is an async method for the background reader to continue syncing new messages. + * + * Note: You should not do any blocking call here. because it will affect + * #{@link SystemTopicBasedTopicPoliciesService#getTopicPoliciesAsync(TopicName)} method to block loading topic. + */ + private void readMorePoliciesAsync(SystemTopicClient.Reader reader) { reader.readNextAsync() .thenAccept(msg -> { refreshTopicPoliciesCache(msg); @@ -428,7 +474,7 @@ private void readMorePolicies(SystemTopicClient.Reader reader) { }) .whenComplete((__, ex) -> { if (ex == null) { - readMorePolicies(reader); + readMorePoliciesAsync(reader); } else { Throwable cause = FutureUtil.unwrapCompletionException(ex); if (cause instanceof PulsarClientException.AlreadyClosedException) { @@ -437,7 +483,7 @@ private void readMorePolicies(SystemTopicClient.Reader reader) { reader.getSystemTopic().getTopicName().getNamespaceObject(), false); } else { log.warn("Read more topic polices exception, read again.", ex); - readMorePolicies(reader); + readMorePoliciesAsync(reader); } } }); @@ -605,7 +651,7 @@ boolean checkReaderIsCached(NamespaceName namespaceName) { } @VisibleForTesting - public Boolean getPoliciesCacheInit(NamespaceName namespaceName) { + public CompletableFuture getPoliciesCacheInit(NamespaceName namespaceName) { return policyCacheInitMap.get(namespaceName); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index c4bcc0c39353c..aa3a6aaeff29f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -22,6 +22,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.BackoffBuilder; @@ -31,6 +32,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.FutureUtil; +import org.jetbrains.annotations.NotNull; /** * Topic policies service. @@ -109,6 +111,32 @@ default CompletableFuture> getTopicPoliciesAsyncWithRetr return response; } + /** + * Asynchronously retrieves topic policies. + * This triggers the Pulsar broker's internal client to load policies from the + * system topic `persistent://tenant/namespace/__change_event`. + * + * @param topicName The name of the topic. + * @param isGlobal Indicates if the policies are global. + * @return A CompletableFuture containing an Optional of TopicPolicies. + * @throws NullPointerException If the topicName is null. + */ + @Nonnull + CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName, boolean isGlobal); + + /** + * Asynchronously retrieves topic policies. + * This triggers the Pulsar broker's internal client to load policies from the + * system topic `persistent://tenant/namespace/__change_event`. + * + * NOTE: If local policies are not available, it will fallback to using topic global policies. + * @param topicName The name of the topic. + * @return A CompletableFuture containing an Optional of TopicPolicies. + * @throws NullPointerException If the topicName is null. + */ + @Nonnull + CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName); + /** * Get policies for a topic without cache async. * @param topicName topic name @@ -162,6 +190,19 @@ public TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) return null; } + @NotNull + @Override + public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName, + boolean isGlobal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @NotNull + @Override + public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName) { + return CompletableFuture.completedFuture(Optional.empty()); + } + @Override public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { return null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 284e50c830286..8dee90af4afb4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -126,6 +126,7 @@ public void initPersistentTopics() throws Exception { @Override @BeforeMethod protected void setup() throws Exception { + conf.setTopicLevelPoliciesEnabled(false); super.internalSetup(); persistentTopics = spy(PersistentTopics.class); persistentTopics.setServletContext(new MockServletContext()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 87471f4972f8d..faf141a5d1cf4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -180,7 +180,7 @@ public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Excep assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); //make sure namespace policy reader is fully started. Awaitility.await().untilAsserted(()-> { - assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject())); + assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject()).isDone()); }); //load the topic. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java new file mode 100644 index 0000000000000..672fc2c95f890 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Test(groups = "broker-admin") +public class TopicPoliciesWithBrokerRestartTest extends MockedPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true) + protected void setup() throws Exception { + super.internalSetup(); + setupDefaultTenantAndNamespace(); + } + + @Override + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + + @Test + public void testRetentionWithBrokerRestart() throws Exception { + final int messages = 1_000; + final int topicNum = 500; + // (1) Init topic + admin.namespaces().createNamespace("public/retention"); + final String topicName = "persistent://public/retention/retention_with_broker_restart"; + admin.topics().createNonPartitionedTopic(topicName); + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.topics().createNonPartitionedTopic(shadowTopicNames); + } + // (2) Set retention + final RetentionPolicies retentionPolicies = new RetentionPolicies(20, 20); + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.topicPolicies().setRetention(shadowTopicNames, retentionPolicies); + } + admin.topicPolicies().setRetention(topicName, retentionPolicies); + // (3) Send messages + @Cleanup + final Producer publisher = pulsarClient.newProducer() + .topic(topicName) + .create(); + for (int i = 0; i < messages; i++) { + publisher.send((i + "").getBytes(StandardCharsets.UTF_8)); + } + // (4) Check configuration + Awaitility.await().untilAsserted(() -> { + final PersistentTopic persistentTopic1 = (PersistentTopic) + pulsar.getBrokerService().getTopic(topicName, true).join().get(); + final ManagedLedgerImpl managedLedger1 = (ManagedLedgerImpl) persistentTopic1.getManagedLedger(); + Assert.assertEquals(managedLedger1.getConfig().getRetentionSizeInMB(), 20); + Assert.assertEquals(managedLedger1.getConfig().getRetentionTimeMillis(), + TimeUnit.MINUTES.toMillis(20)); + }); + // (5) Restart broker + restartBroker(); + // (6) Check configuration again + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.lookups().lookupTopic(shadowTopicNames); + final PersistentTopic persistentTopicTmp = (PersistentTopic) + pulsar.getBrokerService().getTopic(shadowTopicNames, true).join().get(); + final ManagedLedgerImpl managedLedgerTemp = (ManagedLedgerImpl) persistentTopicTmp.getManagedLedger(); + Assert.assertEquals(managedLedgerTemp.getConfig().getRetentionSizeInMB(), 20); + Assert.assertEquals(managedLedgerTemp.getConfig().getRetentionTimeMillis(), + TimeUnit.MINUTES.toMillis(20)); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java index efd8b66d754ac..234af7afa8d09 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java @@ -84,6 +84,8 @@ protected void setup() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); super.internalSetup(); PulsarAdminBuilder pulsarAdminBuilder = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java index 6ffcecbeb9f8b..942a42fa7aaa1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java @@ -60,6 +60,8 @@ public void setup() throws Exception { conf.setAuthorizationEnabled(true); conf.setAuthorizationAllowWildcardsMatching(true); conf.setSuperUserRoles(Sets.newHashSet("super")); + conf.setBrokerClientAuthenticationPlugin(MockAuthentication.class.getName()); + conf.setBrokerClientAuthenticationParameters("user:pass.pass"); internalSetup(); try (PulsarAdmin admin = PulsarAdmin.builder() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java index 0b1726617f71f..25ac59796b02c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java @@ -29,7 +29,10 @@ public class MockAuthentication implements Authentication { private static final Logger log = LoggerFactory.getLogger(MockAuthentication.class); - private final String user; + private String user; + + public MockAuthentication() { + } public MockAuthentication(String user) { this.user = user; @@ -67,6 +70,7 @@ public String getCommandData() { @Override public void configure(Map authParams) { + this.user = authParams.get("user"); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index b16dc27e26586..0f04088e21a03 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -42,6 +42,7 @@ import lombok.Data; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -53,6 +54,8 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; +import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -222,6 +225,22 @@ protected void doInitConf() throws Exception { protected final void init() throws Exception { doInitConf(); + // trying to config the broker internal client + if (conf.getWebServicePortTls().isPresent() + && conf.getAuthenticationProviders().contains(AuthenticationProviderTls.class.getName()) + && !conf.isTlsEnabledWithKeyStore()) { + // enabled TLS + if (conf.getBrokerClientAuthenticationPlugin() == null + || conf.getBrokerClientAuthenticationPlugin().equals(AuthenticationDisabled.class.getName())) { + conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); + conf.setBrokerClientAuthenticationParameters("tlsCertFile:" + BROKER_CERT_FILE_PATH + + ",tlsKeyFile:" + BROKER_KEY_FILE_PATH); + conf.setBrokerClientTlsEnabled(true); + conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setBrokerClientCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setBrokerClientKeyFilePath(BROKER_KEY_FILE_PATH); + } + } startBroker(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java index 951892f4ebfbc..5252407892eea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java @@ -304,6 +304,7 @@ public void testSetRackInfoAndAffinityGroupDuringProduce() throws Exception { bookies[3].getBookieId()); ServiceConfiguration config = new ServiceConfiguration(); + config.setTopicLevelPoliciesEnabled(false); config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config.setClusterName(cluster); config.setWebServicePort(Optional.of(0)); @@ -612,9 +613,9 @@ public void testBookieIsolationWithSecondaryGroup() throws Exception { config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); + config.setTopicLevelPoliciesEnabled(false); config.setAdvertisedAddress("localhost"); config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups); - config.setManagedLedgerDefaultEnsembleSize(2); config.setManagedLedgerDefaultWriteQuorum(2); config.setManagedLedgerDefaultAckQuorum(2); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 31b5bcb23cd98..5b70ff996756e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -141,7 +141,7 @@ public void testGetPolicy() throws ExecutionException, InterruptedException, Top // Wait for all topic policies updated. Awaitility.await().untilAsserted(() -> Assert.assertTrue(systemTopicBasedTopicPoliciesService - .getPoliciesCacheInit(TOPIC1.getNamespaceObject()))); + .getPoliciesCacheInit(TOPIC1.getNamespaceObject()).isDone())); // Assert broker is cache all topic policies Awaitility.await().untilAsserted(() -> @@ -304,8 +304,8 @@ private void prepareData() throws PulsarAdminException { @Test public void testGetPolicyTimeout() throws Exception { SystemTopicBasedTopicPoliciesService service = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); - Awaitility.await().untilAsserted(() -> assertTrue(service.policyCacheInitMap.get(TOPIC1.getNamespaceObject()))); - service.policyCacheInitMap.put(TOPIC1.getNamespaceObject(), false); + Awaitility.await().untilAsserted(() -> assertTrue(service.policyCacheInitMap.get(TOPIC1.getNamespaceObject()).isDone())); + service.policyCacheInitMap.put(TOPIC1.getNamespaceObject(), new CompletableFuture<>()); long start = System.currentTimeMillis(); Backoff backoff = new BackoffBuilder() .setInitialTime(500, TimeUnit.MILLISECONDS) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 41704af0b8bb2..ac2727e33eb33 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -45,6 +45,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -620,7 +621,7 @@ public void testCheckPersistencePolicies() throws Exception { doReturn(policiesService).when(pulsar).getTopicPoliciesService(); TopicPolicies policies = new TopicPolicies(); policies.setRetentionPolicies(retentionPolicies); - doReturn(policies).when(policiesService).getTopicPoliciesIfExists(TopicName.get(topic)); + doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(policiesService).getTopicPoliciesAsync(TopicName.get(topic)); persistentTopic.onUpdate(policies); verify(persistentTopic, times(1)).checkPersistencePolicies(); Awaitility.await().untilAsserted(() -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java index f2631f591217b..a392f150e85c7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.tls.PublicSuffixMatcher; import org.apache.pulsar.common.tls.TlsHostnameVerifier; +import org.assertj.core.util.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -147,6 +148,7 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname // setup broker cert which has CN = "pulsar" different than broker's hostname="localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); + conf.setAuthenticationProviders(Sets.newTreeSet(AuthenticationProviderTls.class.getName())); conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_MIM_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_MIM_SERVER_KEY_FILE_PATH); @@ -188,6 +190,7 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception { // setup broker cert which has CN = "localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); + conf.setAuthenticationProviders(Sets.newTreeSet(AuthenticationProviderTls.class.getName())); conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 0ce3b7df07d1f..ba41848bf2c23 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -119,6 +119,7 @@ protected void cleanup() throws Exception { public void testProducerAndConsumerAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(TestAuthorizationProvider.class.getName()); setup(); @@ -179,6 +180,7 @@ public void testProducerAndConsumerAuthorization() throws Exception { public void testSubscriberPermission() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setEnablePackagesManagement(true); conf.setPackagesManagementStorageProvider(MockedPackagesStorageProvider.class.getName()); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); @@ -369,6 +371,7 @@ public void testSubscriberPermission() throws Exception { public void testClearBacklogPermission() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); @@ -610,6 +613,7 @@ public void testUpdateTopicPropertiesAuthorization() throws Exception { public void testSubscriptionPrefixAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(TestAuthorizationProviderWithSubscriptionPrefix.class.getName()); setup(); @@ -694,6 +698,7 @@ public void testAuthData() throws Exception { public void testPermissionForProducerCreateInitialSubscription() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java index 2fc8aebf64a4a..81d65b192049b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java @@ -195,7 +195,7 @@ protected void setup() throws Exception { Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); conf.setSuperUserRoles(superUserRoles); - + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationEnabled(true); conf.setAuthenticationEnabled(true); Set providersClassNames = Sets.newHashSet(MutualAuthenticationProvider.class.getName()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java index 87f12e6acdcb2..4d5e7deaf7d99 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.slf4j.Logger; @@ -92,6 +93,8 @@ protected void setup() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); conf.setClusterName("test"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java index fdf41c4a6ada1..ba43ee6d6a2dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java @@ -27,7 +27,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -39,6 +41,7 @@ import org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,11 +90,12 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setBrokerClientAuthenticationPlugin(AuthenticationOAuth2.class.getName()); - conf.setBrokerClientAuthenticationParameters("{\n" - + " \"privateKey\": \"" + CREDENTIALS_FILE + "\",\n" - + " \"issuerUrl\": \"" + server.getIssuer() + "\",\n" - + " \"audience\": \"" + audience + "\",\n" - + "}\n"); + final Map oauth2Param = new HashMap<>(); + oauth2Param.put("privateKey", CREDENTIALS_FILE); + oauth2Param.put("issuerUrl", server.getIssuer()); + oauth2Param.put("audience", audience); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(oauth2Param)); conf.setClusterName("test"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java index 8e508b6cf2068..77405e142013a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java @@ -32,6 +32,7 @@ import io.jsonwebtoken.SignatureAlgorithm; import lombok.Cleanup; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; @@ -49,6 +50,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -83,6 +85,7 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @SneakyThrows protected void internalSetUpForBroker() { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); @@ -114,6 +117,25 @@ protected void internalSetUpForBroker() { conf.setAuthenticationProviders(providers); conf.setNumExecutorThreadPoolSize(5); + Set tlsProtocols = Sets.newConcurrentHashSet(); + tlsProtocols.add("TLSv1.3"); + tlsProtocols.add("TLSv1.2"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationKeyStoreTls.class.getName()); + Map authParams = new HashMap<>(); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_TYPE, KEYSTORE_TYPE); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory.getMapper() + .getObjectMapper().writeValueAsString(authParams)); + conf.setBrokerClientTlsEnabled(true); + conf.setBrokerClientTlsEnabledWithKeyStore(true); + conf.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); + conf.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); + conf.setBrokerClientTlsKeyStore(CLIENT_KEYSTORE_FILE_PATH); + conf.setBrokerClientTlsKeyStoreType(KEYSTORE_TYPE); + conf.setBrokerClientTlsKeyStorePassword(CLIENT_KEYSTORE_PW); + conf.setBrokerClientTlsProtocols(tlsProtocols); + } protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java index 76936334eb0ba..b9139dabdf021 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java @@ -85,6 +85,7 @@ public void setup() throws Exception { // set isTcpLookup = true, to use BinaryProtoLookupService to get topics for a pattern. isTcpLookup = true; + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); diff --git a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf index d9411e655ad5b..4e2fd40298354 100644 --- a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf +++ b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf @@ -29,4 +29,5 @@ authenticationEnabled=true authenticationProviders=org.apache.pulsar.MockTokenAuthenticationProvider brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= -loadBalancerOverrideBrokerNicSpeedGbps=2 \ No newline at end of file +loadBalancerOverrideBrokerNicSpeedGbps=2 +topicLevelPoliciesEnabled=false \ No newline at end of file diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java index 8229d929ee5e3..9c8e5197adf1a 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java @@ -168,7 +168,7 @@ protected void setup() throws Exception { conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); // Expires after an hour conf.setBrokerClientAuthenticationParameters( - "entityType:broker,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000)); + "entityType:admin,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000)); Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java index 99af3b1cf6abe..b7cfb87474707 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java @@ -53,7 +53,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("authParam:broker"); + conf.setBrokerClientAuthenticationParameters("authParam:admin"); conf.setAuthenticateOriginalAuthData(true); Set superUserRoles = new HashSet(); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java index 2c8c382b6a5ef..3259cfd95c741 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java @@ -144,7 +144,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("authParam:broker"); + conf.setBrokerClientAuthenticationParameters("authParam:admin"); Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java index e8bb128c8c190..2d97a4b06a856 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java @@ -78,6 +78,8 @@ public class ProxyWithAuthorizationNegTest extends ProducerConsumerBase { protected void setup() throws Exception { // enable tls and auth&auth at broker + conf.setTopicLevelPoliciesEnabled(false); + conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java index 9119ffed4e28f..7b550b7270f37 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -63,6 +64,9 @@ public void setup() throws Exception { conf.setProperties(properties); conf.setSuperUserRoles(Sets.newHashSet(SUPER_USER_ROLE)); conf.setClusterName("c1"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + AuthTokenUtils + .createToken(secretKey, SUPER_USER_ROLE, Optional.empty())); internalSetup(); admin.clusters().createCluster("c1", ClusterData.builder().build()); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 057039edc3be2..49e5ae378342d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -89,6 +89,7 @@ public void setup() throws Exception { "org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder"); brokerEnvs.put("forceDeleteNamespaceAllowed", "true"); brokerEnvs.put("loadBalancerDebugModeEnabled", "true"); + brokerEnvs.put("topicLevelPoliciesEnabled", "false"); brokerEnvs.put("PULSAR_MEM", "-Xmx512M"); spec.brokerEnvs(brokerEnvs); pulsarCluster = PulsarCluster.forSpec(spec); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java index 0a9bb5e19592a..87db46f2bb625 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java @@ -68,6 +68,7 @@ protected void beforeStartCluster() { envMap.put("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(secretKey)); envMap.put("superUserRoles", "admin"); envMap.put("brokerDeleteInactiveTopicsEnabled", "false"); + envMap.put("topicLevelPoliciesEnabled", "false"); for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { brokerContainer.withEnv(envMap); From c4de4ee8bfd98dad577e6776a3c3838a0808b129 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Thu, 28 Sep 2023 22:14:25 +0800 Subject: [PATCH 306/494] [improve] [auto-recovery] Migrate the replication testing from BookKeeper to Pulsar. (#21188) There is no testing for AutoRecovery replication in Pulsar's current test suite, and we need to cover it. So migrate the replication testing from BookKeeper to Pulsar. (cherry picked from commit 9061d8b4e821e3fd03e067f01d5dfcf05bcdced4) --- .../bookkeeper/PulsarLayoutManager.java | 4 +- .../PulsarLedgerAuditorManager.java | 4 +- .../bookkeeper/PulsarLedgerManager.java | 2 +- .../AuditorBookieCheckTaskTest.java | 153 ++ .../replication/AuditorBookieTest.java | 299 ++++ .../AuditorCheckAllLedgersTaskTest.java | 136 ++ .../replication/AuditorLedgerCheckerTest.java | 1058 ++++++++++++++ .../AuditorPeriodicBookieCheckTest.java | 117 ++ .../replication/AuditorPeriodicCheckTest.java | 852 +++++++++++ .../AuditorPlacementPolicyCheckTaskTest.java | 124 ++ .../AuditorPlacementPolicyCheckTest.java | 861 ++++++++++++ .../AuditorReplicasCheckTaskTest.java | 119 ++ .../replication/AuditorReplicasCheckTest.java | 937 +++++++++++++ .../AuditorRollingRestartTest.java | 112 ++ .../replication/AuthAutoRecoveryTest.java | 119 ++ .../replication/AutoRecoveryMainTest.java | 254 ++++ .../replication/BookieAutoRecoveryTest.java | 652 +++++++++ .../replication/BookieLedgerIndexTest.java | 248 ++++ .../replication/ReplicationTestUtil.java | 61 + ...estAutoRecoveryAlongWithBookieServers.java | 117 ++ .../replication/TestReplicationWorker.java | 1249 +++++++++++++++++ 21 files changed, 7473 insertions(+), 5 deletions(-) create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java index a4336b876398a..ee06930b3c880 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLayoutManager.java @@ -30,7 +30,7 @@ import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; -class PulsarLayoutManager implements LayoutManager { +public class PulsarLayoutManager implements LayoutManager { @Getter(AccessLevel.PACKAGE) private final MetadataStoreExtended store; @@ -40,7 +40,7 @@ class PulsarLayoutManager implements LayoutManager { private final String layoutPath; - PulsarLayoutManager(MetadataStoreExtended store, String ledgersRootPath) { + public PulsarLayoutManager(MetadataStoreExtended store, String ledgersRootPath) { this.ledgersRootPath = ledgersRootPath; this.store = store; this.layoutPath = ledgersRootPath + "/" + BookKeeperConstants.LAYOUT_ZNODE; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java index bc35380fec19c..957f26ecd2e80 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java @@ -30,9 +30,9 @@ import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl; @Slf4j -class PulsarLedgerAuditorManager implements LedgerAuditorManager { +public class PulsarLedgerAuditorManager implements LedgerAuditorManager { - private static final String ELECTION_PATH = "leader"; + public static final String ELECTION_PATH = "leader"; private final CoordinationService coordinationService; private final LeaderElection leaderElection; diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java index 59452a3d54db3..b003c656353c0 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerManager.java @@ -377,7 +377,7 @@ public void close() throws IOException { } } - private String getLedgerPath(long ledgerId) { + public String getLedgerPath(long ledgerId) { return this.ledgerRootPath + StringUtils.getHybridHierarchicalLedgerPath(ledgerId); } diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java new file mode 100644 index 0000000000000..d30af4de6a803 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieCheckTaskTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.AssertJUnit.assertTrue; +import com.beust.jcommander.internal.Lists; +import com.beust.jcommander.internal.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.stats.OpStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.versioning.LongVersion; +import org.apache.bookkeeper.versioning.Versioned; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link AuditorBookieCheckTask}. + */ +public class AuditorBookieCheckTaskTest { + + private AuditorStats auditorStats; + private BookKeeperAdmin admin; + private LedgerManager ledgerManager; + private LedgerUnderreplicationManager underreplicationManager; + private BookieLedgerIndexer ledgerIndexer; + private AuditorBookieCheckTask bookieCheckTask; + private final AtomicBoolean shutdownCompleted = new AtomicBoolean(false); + private final AuditorTask.ShutdownTaskHandler shutdownTaskHandler = () -> shutdownCompleted.set(true); + private long startLedgerId = 0; + + @BeforeMethod + public void setup() { + ServerConfiguration conf = mock(ServerConfiguration.class); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + final AuditorStats auditorStats = new AuditorStats(statsLogger); + this.auditorStats = spy(auditorStats); + admin = mock(BookKeeperAdmin.class); + ledgerManager = mock(LedgerManager.class); + underreplicationManager = mock(LedgerUnderreplicationManager.class); + ledgerIndexer = mock(BookieLedgerIndexer.class); + AuditorBookieCheckTask bookieCheckTask1 = new AuditorBookieCheckTask( + conf, this.auditorStats, admin, ledgerManager, underreplicationManager, + shutdownTaskHandler, ledgerIndexer, null, null); + bookieCheckTask = spy(bookieCheckTask1); + } + + @Test + public void testShutdownAuditBookiesException() + throws BKException, ReplicationException.BKAuditException, InterruptedException { + doThrow(new ReplicationException.BKAuditException("test failed")) + .when(bookieCheckTask) + .auditBookies(); + bookieCheckTask.startAudit(true); + + assertTrue("shutdownTaskHandler should be execute.", shutdownCompleted.get()); + } + + @Test + public void testAuditBookies() + throws ReplicationException.UnavailableException, ReplicationException.BKAuditException, BKException { + final String bookieId1 = "127.0.0.1:1000"; + final String bookieId2 = "127.0.0.1:1001"; + final long bookie1LedgersCount = 10; + final long bookie2LedgersCount = 20; + + final Map> bookiesAndLedgers = new HashMap<>(); + bookiesAndLedgers.put(bookieId1, getLedgers(bookie1LedgersCount)); + bookiesAndLedgers.put(bookieId2, getLedgers(bookie2LedgersCount)); + when(ledgerIndexer.getBookieToLedgerIndex()).thenReturn(bookiesAndLedgers); + when(underreplicationManager.isLedgerReplicationEnabled()).thenReturn(true); + + CompletableFuture> metaPromise = new CompletableFuture<>(); + final LongVersion version = mock(LongVersion.class); + final LedgerMetadata metadata = mock(LedgerMetadata.class); + metaPromise.complete(new Versioned<>(metadata, version)); + when(ledgerManager.readLedgerMetadata(anyLong())).thenReturn(metaPromise); + + CompletableFuture markPromise = new CompletableFuture<>(); + markPromise.complete(null); + when(underreplicationManager.markLedgerUnderreplicatedAsync(anyLong(), anyCollection())) + .thenReturn(markPromise); + + OpStatsLogger numUnderReplicatedLedgerStats = mock(OpStatsLogger.class); + when(auditorStats.getNumUnderReplicatedLedger()).thenReturn(numUnderReplicatedLedgerStats); + + final List availableBookies = Lists.newArrayList(); + final List readOnlyBookies = Lists.newArrayList(); + // test bookie1 lost + availableBookies.add(BookieId.parse(bookieId2)); + when(admin.getAvailableBookies()).thenReturn(availableBookies); + when(admin.getReadOnlyBookies()).thenReturn(readOnlyBookies); + bookieCheckTask.startAudit(true); + verify(numUnderReplicatedLedgerStats, times(1)) + .registerSuccessfulValue(eq(bookie1LedgersCount)); + + // test bookie2 lost + numUnderReplicatedLedgerStats = mock(OpStatsLogger.class); + when(auditorStats.getNumUnderReplicatedLedger()).thenReturn(numUnderReplicatedLedgerStats); + availableBookies.clear(); + availableBookies.add(BookieId.parse(bookieId1)); + bookieCheckTask.startAudit(true); + verify(numUnderReplicatedLedgerStats, times(1)) + .registerSuccessfulValue(eq(bookie2LedgersCount)); + + } + + private Set getLedgers(long count) { + final Set ledgers = Sets.newHashSet(); + for (int i = 0; i < count; i++) { + ledgers.add(i + startLedgerId++); + } + return ledgers; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java new file mode 100644 index 0000000000000..9750fb52d41a3 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.AssertJUnit.assertSame; +import static org.testng.AssertJUnit.assertTrue; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerAuditorManager; +import org.apache.zookeeper.ZooKeeper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * This test verifies the auditor bookie scenarios which will be monitoring the + * bookie failures. + */ +public class AuditorBookieTest extends BookKeeperClusterTestCase { + // Depending on the taste, select the amount of logging + // by decommenting one of the two lines below + // private static final Logger LOG = Logger.getRootLogger(); + private static final Logger LOG = LoggerFactory + .getLogger(AuditorBookieTest.class); + private String electionPath; + private HashMap auditorElectors = new HashMap(); + private List zkClients = new LinkedList(); + + public AuditorBookieTest() throws Exception { + super(6); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + electionPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf) + + "/underreplication/" + PulsarLedgerAuditorManager.ELECTION_PATH; + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + startAuditorElectors(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + stopAuditorElectors(); + for (ZooKeeper zk : zkClients) { + zk.close(); + } + zkClients.clear(); + super.tearDown(); + } + + /** + * Test should ensure only one should act as Auditor. Starting/shutdown + * other than auditor bookie shouldn't initiate re-election and multiple + * auditors. + */ + @Test + public void testEnsureOnlySingleAuditor() throws Exception { + BookieServer auditor = verifyAuditor(); + + // shutdown bookie which is not an auditor + int indexOf = indexOfServer(auditor); + int bkIndexDownBookie; + if (indexOf < lastBookieIndex()) { + bkIndexDownBookie = indexOf + 1; + } else { + bkIndexDownBookie = indexOf - 1; + } + shutdownBookie(serverByIndex(bkIndexDownBookie)); + + startNewBookie(); + startNewBookie(); + // grace period for the auditor re-election if any + BookieServer newAuditor = waitForNewAuditor(auditor); + assertSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor); + } + + /** + * Test Auditor crashes should trigger re-election and another bookie should + * take over the auditor ship. + */ + @Test + public void testSuccessiveAuditorCrashes() throws Exception { + BookieServer auditor = verifyAuditor(); + shutdownBookie(auditor); + + BookieServer newAuditor1 = waitForNewAuditor(auditor); + shutdownBookie(newAuditor1); + BookieServer newAuditor2 = waitForNewAuditor(newAuditor1); + assertNotSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor2); + } + + /** + * Test restarting the entire bookie cluster. It shouldn't create multiple + * bookie auditors. + */ + @Test + public void testBookieClusterRestart() throws Exception { + BookieServer auditor = verifyAuditor(); + for (AuditorElector auditorElector : auditorElectors.values()) { + assertTrue("Auditor elector is not running!", auditorElector + .isRunning()); + } + stopBKCluster(); + stopAuditorElectors(); + + startBKCluster(zkUtil.getMetadataServiceUri()); + //startBKCluster(zkUtil.getMetadataServiceUri()) override the base conf metadataServiceUri + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + startAuditorElectors(); + BookieServer newAuditor = waitForNewAuditor(auditor); + assertNotSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor); + } + + /** + * Test the vote is deleting from the ZooKeeper during shutdown. + */ + @Test + public void testShutdown() throws Exception { + BookieServer auditor = verifyAuditor(); + shutdownBookie(auditor); + + // waiting for new auditor + BookieServer newAuditor = waitForNewAuditor(auditor); + assertNotSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor); + + List children = zkc.getChildren(electionPath, false); + for (String child : children) { + byte[] data = zkc.getData(electionPath + '/' + child, false, null); + String bookieIP = new String(data); + String addr = auditor.getBookieId().toString(); + assertFalse("AuditorElection cleanup fails", bookieIP + .contains(addr)); + } + } + + /** + * Test restart of the previous Auditor bookie shouldn't initiate + * re-election and should create new vote after restarting. + */ + @Test + public void testRestartAuditorBookieAfterCrashing() throws Exception { + BookieServer auditor = verifyAuditor(); + + String addr = auditor.getBookieId().toString(); + + // restarting Bookie with same configurations. + ServerConfiguration serverConfiguration = shutdownBookie(auditor); + + auditorElectors.remove(addr); + startBookie(serverConfiguration); + // starting corresponding auditor elector + + if (LOG.isDebugEnabled()) { + LOG.debug("Performing Auditor Election:" + addr); + } + startAuditorElector(addr); + + // waiting for new auditor to come + BookieServer newAuditor = waitForNewAuditor(auditor); + assertNotSame( + "Auditor re-election is not happened for auditor failure!", + auditor, newAuditor); + assertFalse("No relection after old auditor rejoins", auditor + .getBookieId().equals(newAuditor.getBookieId())); + } + + private void startAuditorElector(String addr) throws Exception { + AuditorElector auditorElector = new AuditorElector(addr, + baseConf); + auditorElectors.put(addr, auditorElector); + auditorElector.start(); + if (LOG.isDebugEnabled()) { + LOG.debug("Starting Auditor Elector"); + } + } + + private void startAuditorElectors() throws Exception { + for (BookieId addr : bookieAddresses()) { + startAuditorElector(addr.toString()); + } + } + + private void stopAuditorElectors() throws Exception { + for (AuditorElector auditorElector : auditorElectors.values()) { + auditorElector.shutdown(); + if (LOG.isDebugEnabled()) { + LOG.debug("Stopping Auditor Elector!"); + } + } + } + + private BookieServer verifyAuditor() throws Exception { + List auditors = getAuditorBookie(); + assertEquals("Multiple Bookies acting as Auditor!", 1, auditors + .size()); + if (LOG.isDebugEnabled()) { + LOG.debug("Bookie running as Auditor:" + auditors.get(0)); + } + return auditors.get(0); + } + + private List getAuditorBookie() throws Exception { + List auditors = new LinkedList(); + byte[] data = zkc.getData(electionPath, false, null); + assertNotNull("Auditor election failed", data); + for (int i = 0; i < bookieCount(); i++) { + BookieServer bks = serverByIndex(i); + if (new String(data).contains(bks.getBookieId() + "")) { + auditors.add(bks); + } + } + return auditors; + } + + private ServerConfiguration shutdownBookie(BookieServer bkServer) throws Exception { + int index = indexOfServer(bkServer); + String addr = addressByIndex(index).toString(); + if (LOG.isDebugEnabled()) { + LOG.debug("Shutting down bookie:" + addr); + } + + // shutdown bookie which is an auditor + ServerConfiguration conf = killBookie(index); + + // stopping corresponding auditor elector + auditorElectors.get(addr).shutdown(); + return conf; + } + + private BookieServer waitForNewAuditor(BookieServer auditor) + throws Exception { + BookieServer newAuditor = null; + int retryCount = 8; + while (retryCount > 0) { + try { + List auditors = getAuditorBookie(); + if (auditors.size() > 0) { + newAuditor = auditors.get(0); + if (auditor != newAuditor) { + break; + } + } + } catch (Exception ignore) { + } + + Thread.sleep(500); + retryCount--; + } + assertNotNull( + "New Auditor is not reelected after auditor crashes", + newAuditor); + verifyAuditor(); + return newAuditor; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java new file mode 100644 index 0000000000000..5c0a3f39325e5 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import java.util.LinkedList; +import java.util.List; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.LayoutManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link AuditorCheckAllLedgersTask}. + */ +public class AuditorCheckAllLedgersTaskTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorCheckAllLedgersTaskTest.class); + + private static final int maxNumberOfConcurrentOpenLedgerOperations = 500; + private static final int acquireConcurrentOpenLedgerOperationsTimeoutMSec = 120000; + + private BookKeeperAdmin admin; + private LedgerManager ledgerManager; + private LedgerUnderreplicationManager ledgerUnderreplicationManager; + + public AuditorCheckAllLedgersTaskTest() { + super(3); + baseConf.setPageLimit(1); + baseConf.setAutoRecoveryDaemonEnabled(false); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + final BookKeeper bookKeeper = new BookKeeper(baseClientConf); + admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf)); + + String ledgersRoot = "/ledgers"; + String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", ""); + MetadataStoreExtended store = MetadataStoreExtended.create(storeUri, + MetadataStoreConfig.builder().fsyncEnable(false).build()); + LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot); + PulsarLedgerManagerFactory ledgerManagerFactory = new PulsarLedgerManagerFactory(); + + ClientConfiguration conf = new ClientConfiguration(); + conf.setZkLedgersRootPath(ledgersRoot); + ledgerManagerFactory.initialize(conf, layoutManager, 1); + ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager(); + ledgerManager = ledgerManagerFactory.newLedgerManager(); + + baseConf.setAuditorMaxNumberOfConcurrentOpenLedgerOperations(maxNumberOfConcurrentOpenLedgerOperations); + baseConf.setAuditorAcquireConcurrentOpenLedgerOperationsTimeoutMSec( + acquireConcurrentOpenLedgerOperationsTimeoutMSec); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + if (ledgerManager != null) { + ledgerManager.close(); + } + if (ledgerUnderreplicationManager != null) { + ledgerUnderreplicationManager.close(); + } + if (admin != null) { + admin.close(); + } + super.tearDown(); + } + + @Test + public void testCheckAllLedgers() throws Exception { + // 1. create ledgers + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + // 2. init CheckAllLedgersTask + final TestStatsProvider statsProvider = new TestStatsProvider(); + final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + final AuditorStats auditorStats = new AuditorStats(statsLogger); + + AuditorCheckAllLedgersTask auditorCheckAllLedgersTask = new AuditorCheckAllLedgersTask( + baseConf, auditorStats, admin, ledgerManager, + ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false)); + + // 3. checkAllLedgers + auditorCheckAllLedgersTask.runTask(); + + // 4. verify + assertEquals("CHECK_ALL_LEDGERS_TIME", 1, ((TestStatsProvider.TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.CHECK_ALL_LEDGERS_TIME)).getSuccessCount()); + assertEquals("NUM_LEDGERS_CHECKED", numLedgers, + (long) statsLogger.getCounter(ReplicationStats.NUM_LEDGERS_CHECKED).get()); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java new file mode 100644 index 0000000000000..ffd71f9311f42 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -0,0 +1,1058 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import lombok.Cleanup; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.client.AsyncCallback.AddCallback; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LayoutManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataClientDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.UnderreplicatedLedger; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.MetadataStoreException; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerAuditorManager; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests publishing of under replicated ledgers by the Auditor bookie node when + * corresponding bookies identifes as not running. + */ +public class AuditorLedgerCheckerTest extends BookKeeperClusterTestCase { + + // Depending on the taste, select the amount of logging + // by decommenting one of the two lines below + // private static final Logger LOG = Logger.getRootLogger(); + private static final Logger LOG = LoggerFactory + .getLogger(AuditorLedgerCheckerTest.class); + + private static final byte[] ledgerPassword = "aaa".getBytes(); + private Random rng; // Random Number Generator + + private DigestType digestType; + + private String underreplicatedPath; + private Map auditorElectors = new ConcurrentHashMap<>(); + private LedgerUnderreplicationManager urLedgerMgr; + + private Set urLedgerList; + private String electionPath; + + private List ledgerList; + + public AuditorLedgerCheckerTest() + throws Exception { + this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + } + + AuditorLedgerCheckerTest(String ledgerManagerFactoryClass) + throws Exception { + super(3); + LOG.info("Running test case using ledger manager : " + + ledgerManagerFactoryClass); + this.digestType = DigestType.CRC32; + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); // set ledger manager name + baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactoryClass); + baseClientConf + .setLedgerManagerFactoryClassName(ledgerManagerFactoryClass); + } + + @BeforeMethod + public void setUp() throws Exception { + super.setUp(); + underreplicatedPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseClientConf) + + "/underreplication/ledgers"; + electionPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf) + + "/underreplication/" + PulsarLedgerAuditorManager.ELECTION_PATH; + + String ledgersRoot = "/ledgers"; + String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", ""); + MetadataStoreExtended store = MetadataStoreExtended.create(storeUri, + MetadataStoreConfig.builder().fsyncEnable(false).build()); + LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot); + PulsarLedgerManagerFactory ledgerManagerFactory = new PulsarLedgerManagerFactory(); + ClientConfiguration conf = new ClientConfiguration(); + conf.setZkLedgersRootPath(ledgersRoot); + ledgerManagerFactory.initialize(conf, layoutManager, 1); + urLedgerMgr = ledgerManagerFactory.newLedgerUnderreplicationManager(); + urLedgerMgr.setCheckAllLedgersCTime(System.currentTimeMillis()); + + baseClientConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + startAuditorElectors(); + rng = new Random(System.currentTimeMillis()); // Initialize the Random + urLedgerList = new HashSet(); + ledgerList = new ArrayList(2); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + stopAuditorElectors(); + super.tearDown(); + } + + private void startAuditorElectors() throws Exception { + for (String addr : bookieAddresses().stream().map(Object::toString) + .collect(Collectors.toList())) { + AuditorElector auditorElector = new AuditorElector(addr, baseConf); + auditorElectors.put(addr, auditorElector); + auditorElector.start(); + if (LOG.isDebugEnabled()) { + LOG.debug("Starting Auditor Elector"); + } + } + } + + private void stopAuditorElectors() throws Exception { + for (AuditorElector auditorElector : auditorElectors.values()) { + auditorElector.shutdown(); + if (LOG.isDebugEnabled()) { + LOG.debug("Stopping Auditor Elector!"); + } + } + } + + /** + * Test publishing of under replicated ledgers by the auditor bookie. + */ + @Test + public void testSimpleLedger() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + int bkShutdownIndex = lastBookieIndex(); + String shutdownBookie = shutdownBookie(bkShutdownIndex); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + waitForAuditToComplete(); + underReplicaLatch.await(5, TimeUnit.SECONDS); + Map urLedgerData = getUrLedgerData(urLedgerList); + assertEquals("Missed identifying under replicated ledgers", 1, + urLedgerList.size()); + + /* + * Sample data format present in the under replicated ledger path + * + * {4=replica: "10.18.89.153:5002"} + */ + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie + + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookie)); + } + + /** + * Test once published under replicated ledger should exists even after + * restarting respective bookie. + */ + @Test + public void testRestartBookie() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + LedgerHandle lh2 = createAndAddEntriesToLedger(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Created following ledgers : {}, {}", lh1, lh2); + } + + int bkShutdownIndex = lastBookieIndex(); + ServerConfiguration bookieConf1 = confByIndex(bkShutdownIndex); + String shutdownBookie = shutdownBookie(bkShutdownIndex); + + // restart the failed bookie + startAndAddBookie(bookieConf1); + + waitForLedgerMissingReplicas(lh1.getId(), 10, shutdownBookie); + waitForLedgerMissingReplicas(lh2.getId(), 10, shutdownBookie); + } + + /** + * Test publishing of under replicated ledgers when multiple bookie failures + * one after another. + */ + @Test + public void testMultipleBookieFailures() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + + // failing first bookie + shutdownBookie(lastBookieIndex()); + + // simulate re-replication + doLedgerRereplication(lh1.getId()); + + // failing another bookie + String shutdownBookie = shutdownBookie(lastBookieIndex()); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertTrue("Ledger should be missing second replica", + waitForLedgerMissingReplicas(lh1.getId(), 10, shutdownBookie)); + } + + @Test + public void testToggleLedgerReplication() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + ledgerList.add(lh1.getId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Created following ledgers : " + ledgerList); + } + + // failing another bookie + CountDownLatch urReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + // disabling ledger replication + urLedgerMgr.disableLedgerReplication(); + ArrayList shutdownBookieList = new ArrayList(); + shutdownBookieList.add(shutdownBookie(lastBookieIndex())); + shutdownBookieList.add(shutdownBookie(lastBookieIndex())); + + assertFalse("Ledger replication is not disabled!", urReplicaLatch + .await(1, TimeUnit.SECONDS)); + + // enabling ledger replication + urLedgerMgr.enableLedgerReplication(); + assertTrue("Ledger replication is not enabled!", urReplicaLatch.await( + 5, TimeUnit.SECONDS)); + } + + @Test + public void testDuplicateEnDisableAutoRecovery() throws Exception { + urLedgerMgr.disableLedgerReplication(); + try { + urLedgerMgr.disableLedgerReplication(); + fail("Must throw exception, since AutoRecovery is already disabled"); + } catch (UnavailableException e) { + assertTrue("AutoRecovery is not disabled previously!", + e.getCause().getCause() instanceof MetadataStoreException.BadVersionException); + } + urLedgerMgr.enableLedgerReplication(); + try { + urLedgerMgr.enableLedgerReplication(); + fail("Must throw exception, since AutoRecovery is already enabled"); + } catch (UnavailableException e) { + assertTrue("AutoRecovery is not enabled previously!", + e.getCause().getCause() instanceof MetadataStoreException.NotFoundException); + } + } + + /** + * Test Auditor should consider Readonly bookie as available bookie. Should not publish ur ledgers for + * readonly bookies. + */ + @Test + public void testReadOnlyBookieExclusionFromURLedgersCheck() throws Exception { + LedgerHandle lh = createAndAddEntriesToLedger(); + ledgerList.add(lh.getId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Created following ledgers : " + ledgerList); + } + + int count = ledgerList.size(); + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(count); + + final int bkIndex = 2; + ServerConfiguration bookieConf = confByIndex(bkIndex); + BookieServer bk = serverByIndex(bkIndex); + bookieConf.setReadOnlyModeEnabled(true); + + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) + .get(30, TimeUnit.SECONDS); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for Auditor to finish ledger check."); + } + waitForAuditToComplete(); + assertFalse("latch should not have completed", underReplicaLatch.await(5, TimeUnit.SECONDS)); + } + + /** + * Test Auditor should consider Readonly bookie fail and publish ur ledgers for readonly bookies. + */ + @Test + public void testReadOnlyBookieShutdown() throws Exception { + LedgerHandle lh = createAndAddEntriesToLedger(); + long ledgerId = lh.getId(); + ledgerList.add(ledgerId); + if (LOG.isDebugEnabled()) { + LOG.debug("Created following ledgers : " + ledgerList); + } + + int count = ledgerList.size(); + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(count); + + int bkIndex = lastBookieIndex(); + if (LOG.isDebugEnabled()) { + LOG.debug("Moving bookie {} {} to read only...", bkIndex, serverByIndex(bkIndex)); + } + ServerConfiguration bookieConf = confByIndex(bkIndex); + BookieServer bk = serverByIndex(bkIndex); + bookieConf.setReadOnlyModeEnabled(true); + + ((BookieImpl) bk.getBookie()).getStateManager().doTransitionToReadOnlyMode(); + bkc.waitForReadOnlyBookie(BookieImpl.getBookieId(confByIndex(bkIndex))) + .get(30, TimeUnit.SECONDS); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for Auditor to finish ledger check."); + } + waitForAuditToComplete(); + assertFalse("latch should not have completed", underReplicaLatch.await(1, TimeUnit.SECONDS)); + + String shutdownBookie = shutdownBookie(bkIndex); + + // grace period for publishing the bk-ledger + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + waitForAuditToComplete(); + underReplicaLatch.await(5, TimeUnit.SECONDS); + Map urLedgerData = getUrLedgerData(urLedgerList); + assertEquals("Missed identifying under replicated ledgers", 1, urLedgerList.size()); + + /* + * Sample data format present in the under replicated ledger path + * + * {4=replica: "10.18.89.153:5002"} + */ + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, urLedgerList.contains(ledgerId)); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookie)); + } + + public void testInnerDelayedAuditOfLostBookies() throws Exception { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + // wait for 5 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(5); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + String shutdownBookie = shutDownNonAuditorBookie(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(4, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // wait for another 5 seconds for the ledger to get reported as under replicated + assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie + + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookie)); + } + + /** + * Test publishing of under replicated ledgers by the auditor + * bookie is delayed if LostBookieRecoveryDelay option is set. + */ + @Test + public void testDelayedAuditOfLostBookies() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + testInnerDelayedAuditOfLostBookies(); + } + + /** + * Test publishing of under replicated ledgers by the auditor + * bookie is delayed if LostBookieRecoveryDelay option is set + * and it continues to be delayed even when periodic bookie check + * is set to run every 2 secs. I.e. periodic bookie check doesn't + * override the delay + */ + @Test + public void testDelayedAuditWithPeriodicBookieCheck() throws Exception { + // enable periodic bookie check on a cadence of every 2 seconds. + // this requires us to stop the auditor/auditorElectors, set the + // periodic check interval and restart the auditorElectors + stopAuditorElectors(); + baseConf.setAuditorPeriodicBookieCheckInterval(2); + startAuditorElectors(); + + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + // the delaying of audit should just work despite the fact + // we have enabled periodic bookie check + testInnerDelayedAuditOfLostBookies(); + } + + @Test + public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + // wait for 50 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(50); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + String shutdownBookie = shutDownNonAuditorBookie(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(4, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // set lostBookieRecoveryDelay to 0, so that it triggers AuditTask immediately + urLedgerMgr.setLostBookieRecoveryDelay(0); + + // wait for 1 second for the ledger to get reported as under replicated + assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(1, TimeUnit.SECONDS)); + + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie + + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookie)); + } + + @Test + public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + // wait for 3 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(3); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + String shutdownBookie = shutDownNonAuditorBookie(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // set lostBookieRecoveryDelay to 4, so the pending AuditTask is resheduled + urLedgerMgr.setLostBookieRecoveryDelay(4); + + // since we changed the BookieRecoveryDelay period to 4, the audittask shouldn't have been executed + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // wait for 3 seconds (since we already waited for 2 secs) for the ledger to get reported as under replicated + assertTrue("audit of lost bookie isn't delayed", underReplicaLatch.await(3, TimeUnit.SECONDS)); + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie + + "is not listed in the ledger as missing replica :" + data, + data.contains(shutdownBookie)); + } + + @Test + public void testTriggerAuditorWithNoPendingAuditTask() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + int lostBookieRecoveryDelayConfValue = baseConf.getLostBookieRecoveryDelay(); + Auditor auditorBookiesAuditor = getAuditorBookiesAuditor(); + Future auditTask = auditorBookiesAuditor.getAuditTask(); + int lostBookieRecoveryDelayBeforeChange = auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange(); + assertEquals("auditTask is supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay", + lostBookieRecoveryDelayConfValue, lostBookieRecoveryDelayBeforeChange); + + @Cleanup("shutdown") OrderedScheduler scheduler = OrderedScheduler.newSchedulerBuilder() + .name("test-scheduler") + .numThreads(1) + .build(); + @Cleanup MetadataClientDriver driver = + MetadataDrivers.getClientDriver(URI.create(baseClientConf.getMetadataServiceUri())); + driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.of(zkc)); + + // there is no easy way to validate if the Auditor has executed Audit process (Auditor.startAudit), + // without shuttingdown Bookie. To test if by resetting LostBookieRecoveryDelay it does Auditing + // even when there is no pending AuditTask, following approach is needed. + + // Here we are creating few ledgers ledgermetadata with non-existing bookies as its ensemble. + // When Auditor does audit it recognizes these ledgers as underreplicated and mark them as + // under-replicated, since these bookies are not available. + int numofledgers = 5; + Random rand = new Random(); + for (int i = 0; i < numofledgers; i++) { + ArrayList ensemble = new ArrayList(); + ensemble.add(new BookieSocketAddress("99.99.99.99:9999").toBookieId()); + ensemble.add(new BookieSocketAddress("11.11.11.11:1111").toBookieId()); + ensemble.add(new BookieSocketAddress("88.88.88.88:8888").toBookieId()); + + long ledgerId = (Math.abs(rand.nextLong())) % 100000000; + + LedgerMetadata metadata = LedgerMetadataBuilder.create() + .withId(ledgerId) + .withEnsembleSize(3).withWriteQuorumSize(2).withAckQuorumSize(2) + .withPassword("passwd".getBytes()) + .withDigestType(DigestType.CRC32.toApiDigestType()) + .newEnsembleEntry(0L, ensemble).build(); + + try (LedgerManager lm = driver.getLedgerManagerFactory().newLedgerManager()) { + lm.createLedgerMetadata(ledgerId, metadata).get(2000, TimeUnit.MILLISECONDS); + } + ledgerList.add(ledgerId); + } + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size()); + urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelayBeforeChange); + assertTrue("Audit should be triggered and created ledgers should be marked as underreplicated", + underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("All the ledgers should be marked as underreplicated", ledgerList.size(), urLedgerList.size()); + + auditTask = auditorBookiesAuditor.getAuditTask(); + assertEquals("auditTask is supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to BaseConf's lostBookieRecoveryDelay", + lostBookieRecoveryDelayBeforeChange, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + } + + @Test + public void testTriggerAuditorWithPendingAuditTask() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + Auditor auditorBookiesAuditor = getAuditorBookiesAuditor(); + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + int lostBookieRecoveryDelay = 5; + // wait for 5 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + String shutdownBookie = shutDownNonAuditorBookie(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + Future auditTask = auditorBookiesAuditor.getAuditTask(); + assertNotSame("auditTask is not supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to what we set", + lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + + // set lostBookieRecoveryDelay to 5 (previous value), so that Auditor is triggered immediately + urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); + assertTrue("audit of lost bookie shouldn't be delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("all under replicated ledgers should be identified", ledgerList.size(), + urLedgerList.size()); + + Thread.sleep(100); + auditTask = auditorBookiesAuditor.getAuditTask(); + assertEquals("auditTask is supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to previously set value", + lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + } + + @Test + public void testTriggerAuditorBySettingDelayToZeroWithPendingAuditTask() throws Exception { + // wait for a second so that the initial periodic check finishes + Thread.sleep(1000); + + Auditor auditorBookiesAuditor = getAuditorBookiesAuditor(); + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + final CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList + .size()); + + int lostBookieRecoveryDelay = 5; + // wait for 5 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); + + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + String shutdownBookie = shutDownNonAuditorBookie(); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting for ledgers to be marked as under replicated"); + } + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + Future auditTask = auditorBookiesAuditor.getAuditTask(); + assertNotSame("auditTask is not supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to what we set", + lostBookieRecoveryDelay, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + + // set lostBookieRecoveryDelay to 0, so that Auditor is triggered immediately + urLedgerMgr.setLostBookieRecoveryDelay(0); + assertTrue("audit of lost bookie shouldn't be delayed", underReplicaLatch.await(1, TimeUnit.SECONDS)); + assertEquals("all under replicated ledgers should be identified", ledgerList.size(), + urLedgerList.size()); + + Thread.sleep(100); + auditTask = auditorBookiesAuditor.getAuditTask(); + assertEquals("auditTask is supposed to be null", null, auditTask); + assertEquals( + "lostBookieRecoveryDelayBeforeChange of Auditor should be equal to previously set value", + 0, auditorBookiesAuditor.getLostBookieRecoveryDelayBeforeChange()); + } + + /** + * Test audit of bookies is delayed when one bookie is down. But when + * another one goes down, the audit is started immediately. + */ + @Test + public void testDelayedAuditWithMultipleBookieFailures() throws Exception { + // wait for the periodic bookie check to finish + Thread.sleep(1000); + + // create a ledger with a bunch of entries + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size()); + + // wait for 10 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(10); + + // shutdown a non auditor bookie to avoid an election + String shutdownBookie1 = shutDownNonAuditorBookie(); + + // wait for 3 seconds and there shouldn't be any under replicated ledgers + // because we have delayed the start of audit by 10 seconds + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(3, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // Now shutdown the second non auditor bookie; We want to make sure that + // the history about having delayed recovery remains. Hence we make sure + // we bring down a non auditor bookie. This should cause the audit to take + // place immediately and not wait for the remaining 7 seconds to elapse + String shutdownBookie2 = shutDownNonAuditorBookie(); + + // 2 second grace period for the ledgers to get reported as under replicated + Thread.sleep(2000); + + // If the following checks pass, it means that audit happened + // within 2 seconds of second bookie going down and it didn't + // wait for 7 more seconds. Hence the second bookie failure doesn't + // delay the audit + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie1 + shutdownBookie2 + + " are not listed in the ledger as missing replicas :" + data, + data.contains(shutdownBookie1) && data.contains(shutdownBookie2)); + } + + /** + * Test audit of bookies is delayed during rolling upgrade scenario: + * a bookies goes down and comes up, the next bookie go down and up and so on. + * At any time only one bookie is down. + */ + @Test + public void testDelayedAuditWithRollingUpgrade() throws Exception { + // wait for the periodic bookie check to finish + Thread.sleep(1000); + + // create a ledger with a bunch of entries + LedgerHandle lh1 = createAndAddEntriesToLedger(); + Long ledgerId = lh1.getId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created ledger : " + ledgerId); + } + ledgerList.add(ledgerId); + lh1.close(); + + CountDownLatch underReplicaLatch = registerUrLedgerWatcher(ledgerList.size()); + + // wait for 5 seconds before starting the recovery work when a bookie fails + urLedgerMgr.setLostBookieRecoveryDelay(5); + + // shutdown a non auditor bookie to avoid an election + int idx1 = getShutDownNonAuditorBookieIdx(""); + ServerConfiguration conf1 = confByIndex(idx1); + String shutdownBookie1 = shutdownBookie(idx1); + + // wait for 2 seconds and there shouldn't be any under replicated ledgers + // because we have delayed the start of audit by 5 seconds + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // restart the bookie we shut down above + startAndAddBookie(conf1); + + // Now to simulate the rolling upgrade, bring down a bookie different from + // the one we brought down/up above. + String shutdownBookie2 = shutDownNonAuditorBookie(shutdownBookie1); + + // since the first bookie that was brought down/up has come up, there is only + // one bookie down at this time. Hence the lost bookie check shouldn't start + // immediately; it will start 5 seconds after the second bookie went down + assertFalse("audit of lost bookie isn't delayed", underReplicaLatch.await(2, TimeUnit.SECONDS)); + assertEquals("under replicated ledgers identified when it was not expected", 0, + urLedgerList.size()); + + // wait for a total of 6 seconds(2+4) for the ledgers to get reported as under replicated + Thread.sleep(4000); + + // If the following checks pass, it means that auditing happened + // after lostBookieRecoveryDelay during rolling upgrade as expected + assertTrue("Ledger is not marked as underreplicated:" + ledgerId, + urLedgerList.contains(ledgerId)); + Map urLedgerData = getUrLedgerData(urLedgerList); + String data = urLedgerData.get(ledgerId); + assertTrue("Bookie " + shutdownBookie1 + "wrongly listed as missing the ledger: " + data, + !data.contains(shutdownBookie1)); + assertTrue("Bookie " + shutdownBookie2 + + " is not listed in the ledger as missing replicas :" + data, + data.contains(shutdownBookie2)); + LOG.info("*****************Test Complete"); + } + + private void waitForAuditToComplete() throws Exception { + long endTime = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < endTime) { + Auditor auditor = getAuditorBookiesAuditor(); + if (auditor != null) { + Future task = auditor.submitAuditTask(); + task.get(5, TimeUnit.SECONDS); + return; + } + Thread.sleep(100); + } + throw new TimeoutException("Could not find an audit within 5 seconds"); + } + + /** + * Wait for ledger to be underreplicated, and to be missing all replicas specified. + */ + private boolean waitForLedgerMissingReplicas(Long ledgerId, long secondsToWait, String... replicas) + throws Exception { + for (int i = 0; i < secondsToWait; i++) { + try { + UnderreplicatedLedger data = urLedgerMgr.getLedgerUnreplicationInfo(ledgerId); + boolean all = true; + for (String r : replicas) { + all = all && data.getReplicaList().contains(r); + } + if (all) { + return true; + } + } catch (Exception e) { + // may not find node + } + Thread.sleep(1000); + } + return false; + } + + private CountDownLatch registerUrLedgerWatcher(int count) + throws KeeperException, InterruptedException { + final CountDownLatch underReplicaLatch = new CountDownLatch(count); + for (Long ledgerId : ledgerList) { + Watcher urLedgerWatcher = new ChildWatcher(underReplicaLatch); + String znode = ZkLedgerUnderreplicationManager.getUrLedgerZnode(underreplicatedPath, + ledgerId); + zkc.exists(znode, urLedgerWatcher); + } + return underReplicaLatch; + } + + private void doLedgerRereplication(Long... ledgerIds) + throws UnavailableException { + for (int i = 0; i < ledgerIds.length; i++) { + long lid = urLedgerMgr.getLedgerToRereplicate(); + assertTrue("Received unexpected ledgerid", Arrays.asList(ledgerIds).contains(lid)); + urLedgerMgr.markLedgerReplicated(lid); + urLedgerMgr.releaseUnderreplicatedLedger(lid); + } + } + + private String shutdownBookie(int bkShutdownIndex) throws Exception { + BookieServer bkServer = serverByIndex(bkShutdownIndex); + String bookieAddr = bkServer.getBookieId().toString(); + if (LOG.isInfoEnabled()) { + LOG.info("Shutting down bookie:" + bookieAddr); + } + killBookie(bkShutdownIndex); + auditorElectors.get(bookieAddr).shutdown(); + auditorElectors.remove(bookieAddr); + return bookieAddr; + } + + private LedgerHandle createAndAddEntriesToLedger() throws BKException, + InterruptedException { + int numEntriesToWrite = 100; + // Create a ledger + LedgerHandle lh = bkc.createLedger(digestType, ledgerPassword); + LOG.info("Ledger ID: " + lh.getId()); + addEntry(numEntriesToWrite, lh); + return lh; + } + + private void addEntry(int numEntriesToWrite, LedgerHandle lh) + throws InterruptedException, BKException { + final CountDownLatch completeLatch = new CountDownLatch(numEntriesToWrite); + final AtomicInteger rc = new AtomicInteger(BKException.Code.OK); + + for (int i = 0; i < numEntriesToWrite; i++) { + ByteBuffer entry = ByteBuffer.allocate(4); + entry.putInt(rng.nextInt(Integer.MAX_VALUE)); + entry.position(0); + lh.asyncAddEntry(entry.array(), new AddCallback() { + public void addComplete(int rc2, LedgerHandle lh, long entryId, Object ctx) { + rc.compareAndSet(BKException.Code.OK, rc2); + completeLatch.countDown(); + } + }, null); + } + completeLatch.await(); + if (rc.get() != BKException.Code.OK) { + throw BKException.create(rc.get()); + } + + } + + private Map getUrLedgerData(Set urLedgerList) + throws KeeperException, InterruptedException { + Map urLedgerData = new HashMap(); + for (Long ledgerId : urLedgerList) { + String znode = ZkLedgerUnderreplicationManager.getUrLedgerZnode(underreplicatedPath, + ledgerId); + byte[] data = zkc.getData(znode, false, null); + urLedgerData.put(ledgerId, new String(data)); + } + return urLedgerData; + } + + private class ChildWatcher implements Watcher { + private final CountDownLatch underReplicaLatch; + + public ChildWatcher(CountDownLatch underReplicaLatch) { + this.underReplicaLatch = underReplicaLatch; + } + + @Override + public void process(WatchedEvent event) { + LOG.info("Received notification for the ledger path : " + + event.getPath()); + for (Long ledgerId : ledgerList) { + if (event.getPath().contains(ledgerId + "")) { + urLedgerList.add(ledgerId); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Count down and waiting for next notification"); + } + // count down and waiting for next notification + underReplicaLatch.countDown(); + } + } + + private BookieServer getAuditorBookie() throws Exception { + List auditors = new LinkedList(); + byte[] data = zkc.getData(electionPath, false, null); + assertNotNull("Auditor election failed", data); + for (int i = 0; i < bookieCount(); i++) { + BookieId bookieId = addressByIndex(i); + if (new String(data).contains(bookieId + "")) { + auditors.add(serverByIndex(i)); + } + } + assertEquals("Multiple Bookies acting as Auditor!", 1, auditors + .size()); + return auditors.get(0); + } + + private Auditor getAuditorBookiesAuditor() throws Exception { + BookieServer auditorBookieServer = getAuditorBookie(); + String bookieAddr = auditorBookieServer.getBookieId().toString(); + return auditorElectors.get(bookieAddr).auditor; + } + + private String shutDownNonAuditorBookie() throws Exception { + // shutdown bookie which is not an auditor + int indexOf = indexOfServer(getAuditorBookie()); + int bkIndexDownBookie; + if (indexOf < lastBookieIndex()) { + bkIndexDownBookie = indexOf + 1; + } else { + bkIndexDownBookie = indexOf - 1; + } + return shutdownBookie(bkIndexDownBookie); + } + + private int getShutDownNonAuditorBookieIdx(String exclude) throws Exception { + // shutdown bookie which is not an auditor + int indexOf = indexOfServer(getAuditorBookie()); + int bkIndexDownBookie = 0; + for (int i = 0; i <= lastBookieIndex(); i++) { + if (i == indexOf || addressByIndex(i).toString().equals(exclude)) { + continue; + } + bkIndexDownBookie = i; + break; + } + return bkIndexDownBookie; + } + + private String shutDownNonAuditorBookie(String exclude) throws Exception { + return shutdownBookie(getShutDownNonAuditorBookieIdx(exclude)); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java new file mode 100644 index 0000000000000..4acb207570a2d --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithLedgerManagerFactory; +import static org.testng.AssertJUnit.assertEquals; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.UncheckedExecutionException; +import lombok.Cleanup; +import org.apache.bookkeeper.client.ClientUtil; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.conf.TestBKConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * This test verifies that the period check on the auditor + * will pick up on missing data in the client. + */ +public class AuditorPeriodicBookieCheckTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorPeriodicBookieCheckTest.class); + + private AuditorElector auditorElector = null; + + private static final int CHECK_INTERVAL = 1; // run every second + + public AuditorPeriodicBookieCheckTest() throws Exception { + super(3); + baseConf.setPageLimit(1); // to make it easy to push ledger out of cache + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + + ServerConfiguration conf = TestBKConfiguration.newServerConfiguration(); + conf.setAuditorPeriodicBookieCheckInterval(CHECK_INTERVAL); + + conf.setMetadataServiceUri( + metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + conf.setProperty("clientConnectTimeoutMillis", 500); + String addr = addressByIndex(0).toString(); + + auditorElector = new AuditorElector(addr, conf); + auditorElector.start(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + auditorElector.shutdown(); + super.tearDown(); + } + + /** + * Test that the periodic bookie checker works. + */ + @Test + public void testPeriodicBookieCheckInterval() throws Exception { + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + runFunctionWithLedgerManagerFactory(confByIndex(0), mFactory -> { + try (LedgerManager ledgerManager = mFactory.newLedgerManager()) { + @Cleanup final LedgerUnderreplicationManager underReplicationManager = + mFactory.newLedgerUnderreplicationManager(); + long ledgerId = 12345L; + ClientUtil.setupLedger(bkc.getLedgerManager(), ledgerId, + LedgerMetadataBuilder.create().withEnsembleSize(3) + .withWriteQuorumSize(3).withAckQuorumSize(3) + .newEnsembleEntry(0L, Lists.newArrayList( + new BookieSocketAddress("192.0.2.1", 1000).toBookieId(), + getBookie(0), + getBookie(1)))); + long underReplicatedLedger = -1; + for (int i = 0; i < 10; i++) { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger != -1) { + break; + } + Thread.sleep(CHECK_INTERVAL * 1000); + } + assertEquals("Ledger should be under replicated", ledgerId, underReplicatedLedger); + } catch (Exception e) { + throw new UncheckedExecutionException(e.getMessage(), e); + } + return null; + }); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java index c761d46c62266..e3b0aa37d7e55 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java @@ -18,20 +18,42 @@ */ package org.apache.bookkeeper.replication; +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertFalse; import static org.testng.AssertJUnit.assertNotSame; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; import io.netty.buffer.ByteBuf; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilenameFilter; import java.io.IOException; import java.net.URI; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.BookieAccessor; import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.bookie.IndexPersistenceMgr; import org.apache.bookkeeper.bookie.TestBookieImpl; +import org.apache.bookkeeper.client.AsyncCallback.AddCallback; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.BookKeeperAdmin; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.meta.LedgerManagerFactory; @@ -40,8 +62,15 @@ import org.apache.bookkeeper.meta.MetadataDrivers; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.WriteCallback; +import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; +import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; +import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterTest; @@ -112,6 +141,829 @@ public void tearDown() throws Exception { super.tearDown(); } + /** + * test that the periodic checking will detect corruptions in + * the bookie entry log. + */ + @Test + public void testEntryLogCorruption() throws Exception { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + underReplicationManager.disableLedgerReplication(); + + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + long ledgerId = lh.getId(); + for (int i = 0; i < 100; i++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + + BookieAccessor.forceFlush((BookieImpl) serverByIndex(0).getBookie()); + + + File ledgerDir = confByIndex(0).getLedgerDirs()[0]; + ledgerDir = BookieImpl.getCurrentDirectory(ledgerDir); + // corrupt of entryLogs + File[] entryLogs = ledgerDir.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.endsWith(".log"); + } + }); + ByteBuffer junk = ByteBuffer.allocate(1024 * 1024); + for (File f : entryLogs) { + FileOutputStream out = new FileOutputStream(f); + out.getChannel().write(junk); + out.close(); + } + restartBookies(); // restart to clear read buffers + + underReplicationManager.enableLedgerReplication(); + long underReplicatedLedger = -1; + for (int i = 0; i < 10; i++) { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger != -1) { + break; + } + Thread.sleep(CHECK_INTERVAL * 1000); + } + assertEquals("Ledger should be under replicated", ledgerId, underReplicatedLedger); + underReplicationManager.close(); + } + + /** + * test that the period checker will detect corruptions in + * the bookie index files. + */ + @Test + public void testIndexCorruption() throws Exception { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + long ledgerToCorrupt = lh.getId(); + for (int i = 0; i < 100; i++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + + // push ledgerToCorrupt out of page cache (bookie is configured to only use 1 page) + lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + for (int i = 0; i < 100; i++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + + BookieAccessor.forceFlush((BookieImpl) serverByIndex(0).getBookie()); + + File ledgerDir = confByIndex(0).getLedgerDirs()[0]; + ledgerDir = BookieImpl.getCurrentDirectory(ledgerDir); + + // corrupt of entryLogs + File index = new File(ledgerDir, IndexPersistenceMgr.getLedgerName(ledgerToCorrupt)); + LOG.info("file to corrupt{}", index); + ByteBuffer junk = ByteBuffer.allocate(1024 * 1024); + FileOutputStream out = new FileOutputStream(index); + out.getChannel().write(junk); + out.close(); + + long underReplicatedLedger = -1; + for (int i = 0; i < 15; i++) { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger != -1) { + break; + } + Thread.sleep(CHECK_INTERVAL * 1000); + } + assertEquals("Ledger should be under replicated", ledgerToCorrupt, underReplicatedLedger); + underReplicationManager.close(); + } + + /** + * Test that the period checker will not run when auto replication has been disabled. + */ + @Test + public void testPeriodicCheckWhenDisabled() throws Exception { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + final int numLedgers = 10; + final int numMsgs = 2; + final CountDownLatch completeLatch = new CountDownLatch(numMsgs * numLedgers); + final AtomicInteger rc = new AtomicInteger(BKException.Code.OK); + + List lhs = new ArrayList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + lhs.add(lh); + for (int j = 0; j < 2; j++) { + lh.asyncAddEntry("testdata".getBytes(), new AddCallback() { + public void addComplete(int rc2, LedgerHandle lh, long entryId, Object ctx) { + if (rc.compareAndSet(BKException.Code.OK, rc2)) { + LOG.info("Failed to add entry : {}", BKException.getMessage(rc2)); + } + completeLatch.countDown(); + } + }, null); + } + } + completeLatch.await(); + if (rc.get() != BKException.Code.OK) { + throw BKException.create(rc.get()); + } + + for (LedgerHandle lh : lhs) { + lh.close(); + } + + underReplicationManager.disableLedgerReplication(); + + final AtomicInteger numReads = new AtomicInteger(0); + ServerConfiguration conf = killBookie(0); + + Bookie deadBookie = new TestBookieImpl(conf) { + @Override + public ByteBuf readEntry(long ledgerId, long entryId) + throws IOException, NoLedgerException { + // we want to disable during checking + numReads.incrementAndGet(); + throw new IOException("Fake I/O exception"); + } + }; + startAndAddBookie(conf, deadBookie); + + Thread.sleep(CHECK_INTERVAL * 2000); + assertEquals("Nothing should have tried to read", 0, numReads.get()); + underReplicationManager.enableLedgerReplication(); + Thread.sleep(CHECK_INTERVAL * 2000); // give it time to run + + underReplicationManager.disableLedgerReplication(); + // give it time to stop, from this point nothing new should be marked + Thread.sleep(CHECK_INTERVAL * 2000); + + int numUnderreplicated = 0; + long underReplicatedLedger = -1; + do { + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + if (underReplicatedLedger == -1) { + break; + } + numUnderreplicated++; + + underReplicationManager.markLedgerReplicated(underReplicatedLedger); + } while (underReplicatedLedger != -1); + + Thread.sleep(CHECK_INTERVAL * 2000); // give a chance to run again (it shouldn't, it's disabled) + + // ensure that nothing is marked as underreplicated + underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate(); + assertEquals("There should be no underreplicated ledgers", -1, underReplicatedLedger); + + LOG.info("{} of {} ledgers underreplicated", numUnderreplicated, numUnderreplicated); + assertTrue("All should be underreplicated", + numUnderreplicated <= numLedgers && numUnderreplicated > 0); + } + + /** + * Test that the period check will succeed if a ledger is deleted midway. + */ + @Test + public void testPeriodicCheckWhenLedgerDeleted() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + try (final Auditor auditor = new Auditor( + BookieImpl.getBookieId(confByIndex(0)).toString(), + confByIndex(0), NullStatsLogger.INSTANCE)) { + final AtomicBoolean exceptionCaught = new AtomicBoolean(false); + final CountDownLatch latch = new CountDownLatch(1); + Thread t = new Thread() { + public void run() { + try { + latch.countDown(); + for (int i = 0; i < numLedgers; i++) { + ((AuditorCheckAllLedgersTask) auditor.auditorCheckAllLedgersTask).checkAllLedgers(); + } + } catch (Exception e) { + LOG.error("Caught exception while checking all ledgers", e); + exceptionCaught.set(true); + } + } + }; + t.start(); + latch.await(); + for (Long id : ids) { + bkc.deleteLedger(id); + } + t.join(); + assertFalse("Shouldn't have thrown exception", exceptionCaught.get()); + } + } + + @Test + public void testGetLedgerFromZookeeperThrottled() throws Exception { + final int numberLedgers = 30; + + // write ledgers into bookkeeper cluster + try { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + for (int i = 0; i < numberLedgers; ++i) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + for (int j = 0; j < 5; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + } catch (InterruptedException | BKException e) { + LOG.error("Failed to shutdown auditor elector or write data to ledgers ", e); + fail(); + } + + // create auditor and call `checkAllLedgers` + ServerConfiguration configuration = confByIndex(0); + configuration.setAuditorMaxNumberOfConcurrentOpenLedgerOperations(10); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + Counter numLedgersChecked = statsLogger + .getCounter(ReplicationStats.NUM_LEDGERS_CHECKED); + Auditor auditor = new Auditor(BookieImpl.getBookieId(configuration).toString(), + configuration, statsLogger); + + try { + ((AuditorCheckAllLedgersTask) auditor.auditorCheckAllLedgersTask).checkAllLedgers(); + assertEquals("NUM_LEDGERS_CHECKED", numberLedgers, (long) numLedgersChecked.get()); + } catch (Exception e) { + LOG.error("Caught exception while checking all ledgers ", e); + fail(); + } + } + + @Test + public void testInitialDelayOfCheckAllLedgers() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + validateInitialDelayOfCheckAllLedgers(urm, -1, 1000, servConf, bkc); + validateInitialDelayOfCheckAllLedgers(urm, 999, 1000, servConf, bkc); + validateInitialDelayOfCheckAllLedgers(urm, 1001, 1000, servConf, bkc); + } + + void validateInitialDelayOfCheckAllLedgers(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs, + long auditorPeriodicCheckInterval, ServerConfiguration servConf, + BookKeeper bkc) + throws UnavailableException, UnknownHostException, InterruptedException { + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger checkAllLedgersStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.CHECK_ALL_LEDGERS_TIME); + servConf.setAuditorPeriodicCheckInterval(auditorPeriodicCheckInterval); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, + statsLogger, null); + CountDownLatch latch = auditor.getLatch(); + assertEquals("CHECK_ALL_LEDGERS_TIME SuccessCount", 0, checkAllLedgersStatsLogger.getSuccessCount()); + long curTimeBeforeStart = System.currentTimeMillis(); + long checkAllLedgersCTime = -1; + long initialDelayInMsecs = -1; + long nextExpectedCheckAllLedgersExecutionTime = -1; + long bufferTimeInMsecs = 12000L; + if (timeSinceLastExecutedInSecs == -1) { + /* + * if we are setting checkAllLedgersCTime to -1, it means that + * checkAllLedgers hasn't run before. So initialDelay for + * checkAllLedgers should be 0. + */ + checkAllLedgersCTime = -1; + initialDelayInMsecs = 0; + } else { + checkAllLedgersCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L; + initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicCheckInterval ? 0 + : (auditorPeriodicCheckInterval - timeSinceLastExecutedInSecs) * 1000L; + } + /* + * next checkAllLedgers should happen atleast after + * nextExpectedCheckAllLedgersExecutionTime. + */ + nextExpectedCheckAllLedgersExecutionTime = curTimeBeforeStart + initialDelayInMsecs; + + urm.setCheckAllLedgersCTime(checkAllLedgersCTime); + auditor.start(); + /* + * since auditorPeriodicCheckInterval are higher values (in the order of + * 100s of seconds), its ok bufferTimeInMsecs to be ` 10 secs. + */ + assertTrue("checkAllLedgers should have executed with initialDelay " + initialDelayInMsecs, + latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS)); + for (int i = 0; i < 10; i++) { + Thread.sleep(100); + if (checkAllLedgersStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("CHECK_ALL_LEDGERS_TIME SuccessCount", 1, checkAllLedgersStatsLogger.getSuccessCount()); + long currentCheckAllLedgersCTime = urm.getCheckAllLedgersCTime(); + assertTrue( + "currentCheckAllLedgersCTime: " + currentCheckAllLedgersCTime + + " should be greater than nextExpectedCheckAllLedgersExecutionTime: " + + nextExpectedCheckAllLedgersExecutionTime, + currentCheckAllLedgersCTime > nextExpectedCheckAllLedgersExecutionTime); + assertTrue( + "currentCheckAllLedgersCTime: " + currentCheckAllLedgersCTime + + " should be lesser than nextExpectedCheckAllLedgersExecutionTime+bufferTimeInMsecs: " + + (nextExpectedCheckAllLedgersExecutionTime + bufferTimeInMsecs), + currentCheckAllLedgersCTime < (nextExpectedCheckAllLedgersExecutionTime + bufferTimeInMsecs)); + auditor.close(); + } + + @Test + public void testInitialDelayOfPlacementPolicyCheck() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + validateInitialDelayOfPlacementPolicyCheck(urm, -1, 1000, servConf, bkc); + validateInitialDelayOfPlacementPolicyCheck(urm, 999, 1000, servConf, bkc); + validateInitialDelayOfPlacementPolicyCheck(urm, 1001, 1000, servConf, bkc); + } + + void validateInitialDelayOfPlacementPolicyCheck(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs, + long auditorPeriodicPlacementPolicyCheckInterval, + ServerConfiguration servConf, BookKeeper bkc) + throws UnavailableException, UnknownHostException, InterruptedException { + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger placementPolicyCheckStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(auditorPeriodicPlacementPolicyCheckInterval); + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, + statsLogger, null); + CountDownLatch latch = auditor.getLatch(); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount()); + long curTimeBeforeStart = System.currentTimeMillis(); + long placementPolicyCheckCTime = -1; + long initialDelayInMsecs = -1; + long nextExpectedPlacementPolicyCheckExecutionTime = -1; + long bufferTimeInMsecs = 20000L; + if (timeSinceLastExecutedInSecs == -1) { + /* + * if we are setting placementPolicyCheckCTime to -1, it means that + * placementPolicyCheck hasn't run before. So initialDelay for + * placementPolicyCheck should be 0. + */ + placementPolicyCheckCTime = -1; + initialDelayInMsecs = 0; + } else { + placementPolicyCheckCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L; + initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicPlacementPolicyCheckInterval ? 0 + : (auditorPeriodicPlacementPolicyCheckInterval - timeSinceLastExecutedInSecs) * 1000L; + } + /* + * next placementPolicyCheck should happen atleast after + * nextExpectedPlacementPolicyCheckExecutionTime. + */ + nextExpectedPlacementPolicyCheckExecutionTime = curTimeBeforeStart + initialDelayInMsecs; + + urm.setPlacementPolicyCheckCTime(placementPolicyCheckCTime); + auditor.start(); + /* + * since auditorPeriodicPlacementPolicyCheckInterval are higher values (in the + * order of 100s of seconds), its ok bufferTimeInMsecs to be ` 20 secs. + */ + assertTrue("placementPolicyCheck should have executed with initialDelay " + initialDelayInMsecs, + latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, placementPolicyCheckStatsLogger.getSuccessCount()); + long currentPlacementPolicyCheckCTime = urm.getPlacementPolicyCheckCTime(); + assertTrue( + "currentPlacementPolicyCheckCTime: " + currentPlacementPolicyCheckCTime + + " should be greater than nextExpectedPlacementPolicyCheckExecutionTime: " + + nextExpectedPlacementPolicyCheckExecutionTime, + currentPlacementPolicyCheckCTime > nextExpectedPlacementPolicyCheckExecutionTime); + assertTrue( + "currentPlacementPolicyCheckCTime: " + currentPlacementPolicyCheckCTime + + " should be lesser than nextExpectedPlacementPolicyCheckExecutionTime+bufferTimeInMsecs: " + + (nextExpectedPlacementPolicyCheckExecutionTime + bufferTimeInMsecs), + currentPlacementPolicyCheckCTime < (nextExpectedPlacementPolicyCheckExecutionTime + bufferTimeInMsecs)); + auditor.close(); + } + + @Test + public void testInitialDelayOfReplicasCheck() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + LedgerHandle lh = bkc.createLedger(3, 2, DigestType.CRC32, "passwd".getBytes()); + for (int j = 0; j < 5; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + + long ledgerId = 100000L; + lh = bkc.createLedgerAdv(ledgerId, 3, 2, 2, DigestType.CRC32, "passwd".getBytes(), null); + lh.close(); + + ledgerId = 100001234L; + lh = bkc.createLedgerAdv(ledgerId, 3, 3, 2, DigestType.CRC32, "passwd".getBytes(), null); + for (int j = 0; j < 4; j++) { + lh.addEntry(j, "testdata".getBytes()); + } + lh.close(); + + ledgerId = 991234L; + lh = bkc.createLedgerAdv(ledgerId, 3, 2, 2, DigestType.CRC32, "passwd".getBytes(), null); + lh.addEntry(0, "testdata".getBytes()); + lh.close(); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + validateInitialDelayOfReplicasCheck(urm, -1, 1000, servConf, bkc); + validateInitialDelayOfReplicasCheck(urm, 999, 1000, servConf, bkc); + validateInitialDelayOfReplicasCheck(urm, 1001, 1000, servConf, bkc); + } + + void validateInitialDelayOfReplicasCheck(LedgerUnderreplicationManager urm, long timeSinceLastExecutedInSecs, + long auditorPeriodicReplicasCheckInterval, ServerConfiguration servConf, + BookKeeper bkc) + throws UnavailableException, UnknownHostException, InterruptedException { + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger replicasCheckStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME); + servConf.setAuditorPeriodicReplicasCheckInterval(auditorPeriodicReplicasCheckInterval); + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, + statsLogger, null); + CountDownLatch latch = auditor.getLatch(); + assertEquals("REPLICAS_CHECK_TIME SuccessCount", 0, replicasCheckStatsLogger.getSuccessCount()); + long curTimeBeforeStart = System.currentTimeMillis(); + long replicasCheckCTime = -1; + long initialDelayInMsecs = -1; + long nextExpectedReplicasCheckExecutionTime = -1; + long bufferTimeInMsecs = 20000L; + if (timeSinceLastExecutedInSecs == -1) { + /* + * if we are setting replicasCheckCTime to -1, it means that + * replicasCheck hasn't run before. So initialDelay for + * replicasCheck should be 0. + */ + replicasCheckCTime = -1; + initialDelayInMsecs = 0; + } else { + replicasCheckCTime = curTimeBeforeStart - timeSinceLastExecutedInSecs * 1000L; + initialDelayInMsecs = timeSinceLastExecutedInSecs > auditorPeriodicReplicasCheckInterval ? 0 + : (auditorPeriodicReplicasCheckInterval - timeSinceLastExecutedInSecs) * 1000L; + } + /* + * next replicasCheck should happen atleast after + * nextExpectedReplicasCheckExecutionTime. + */ + nextExpectedReplicasCheckExecutionTime = curTimeBeforeStart + initialDelayInMsecs; + + urm.setReplicasCheckCTime(replicasCheckCTime); + auditor.start(); + /* + * since auditorPeriodicReplicasCheckInterval are higher values (in the + * order of 100s of seconds), its ok bufferTimeInMsecs to be ` 20 secs. + */ + assertTrue("replicasCheck should have executed with initialDelay " + initialDelayInMsecs, + latch.await(initialDelayInMsecs + bufferTimeInMsecs, TimeUnit.MILLISECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (replicasCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("REPLICAS_CHECK_TIME SuccessCount", 1, replicasCheckStatsLogger.getSuccessCount()); + long currentReplicasCheckCTime = urm.getReplicasCheckCTime(); + assertTrue( + "currentReplicasCheckCTime: " + currentReplicasCheckCTime + + " should be greater than nextExpectedReplicasCheckExecutionTime: " + + nextExpectedReplicasCheckExecutionTime, + currentReplicasCheckCTime > nextExpectedReplicasCheckExecutionTime); + assertTrue( + "currentReplicasCheckCTime: " + currentReplicasCheckCTime + + " should be lesser than nextExpectedReplicasCheckExecutionTime+bufferTimeInMsecs: " + + (nextExpectedReplicasCheckExecutionTime + bufferTimeInMsecs), + currentReplicasCheckCTime < (nextExpectedReplicasCheckExecutionTime + bufferTimeInMsecs)); + auditor.close(); + } + + @Test + public void testDelayBookieAuditOfCheckAllLedgers() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + Counter numBookieAuditsDelayed = + statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); + TestOpStatsLogger underReplicatedLedgerTotalSizeStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.UNDER_REPLICATED_LEDGERS_TOTAL_SIZE); + + servConf.setAuditorPeriodicCheckInterval(1); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE); + + urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE); + + AtomicBoolean canRun = new AtomicBoolean(false); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, + false, statsLogger, canRun); + final CountDownLatch latch = auditor.getLatch(); + + auditor.start(); + + killBookie(addressByIndex(0)); + + Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get())); + final Future auditTask = auditor.auditTask; + assertTrue(auditTask != null && !auditTask.isDone()); + + canRun.set(true); + + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(auditor.auditTask.equals(auditTask) + && auditor.auditTask != null && !auditor.auditTask.isDone()); + // wrong num is numLedgers, right num is 0 + assertEquals("UNDER_REPLICATED_LEDGERS_TOTAL_SIZE", + 0, + underReplicatedLedgerTotalSizeStatsLogger.getSuccessCount()); + + auditor.close(); + } + + @Test + public void testDelayBookieAuditOfPlacementPolicy() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + Counter numBookieAuditsDelayed = + statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); + TestOpStatsLogger placementPolicyCheckTime = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1); + servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE); + + urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE); + + AtomicBoolean canRun = new AtomicBoolean(false); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, + false, statsLogger, canRun); + final CountDownLatch latch = auditor.getLatch(); + + auditor.start(); + + killBookie(addressByIndex(0)); + + Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get())); + final Future auditTask = auditor.auditTask; + assertTrue(auditTask != null && !auditTask.isDone()); + assertEquals("PLACEMENT_POLICY_CHECK_TIME", 0, placementPolicyCheckTime.getSuccessCount()); + + canRun.set(true); + + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(auditor.auditTask.equals(auditTask) + && auditor.auditTask != null && !auditor.auditTask.isDone()); + // wrong successCount is > 0, right successCount is = 0 + assertEquals("PLACEMENT_POLICY_CHECK_TIME", 0, placementPolicyCheckTime.getSuccessCount()); + + auditor.close(); + } + + @Test + public void testDelayBookieAuditOfReplicasCheck() throws Exception { + for (AuditorElector e : auditorElectors.values()) { + e.shutdown(); + } + + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + Counter numBookieAuditsDelayed = + statsLogger.getCounter(ReplicationStats.NUM_BOOKIE_AUDITS_DELAYED); + TestOpStatsLogger replicasCheckTime = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME); + + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(Long.MAX_VALUE); + servConf.setAuditorPeriodicReplicasCheckInterval(1); + + urm.setLostBookieRecoveryDelay(Integer.MAX_VALUE); + + AtomicBoolean canRun = new AtomicBoolean(false); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, + false, statsLogger, canRun); + final CountDownLatch latch = auditor.getLatch(); + + auditor.start(); + + killBookie(addressByIndex(0)); + + Awaitility.await().untilAsserted(() -> assertEquals(1, (long) numBookieAuditsDelayed.get())); + final Future auditTask = auditor.auditTask; + assertTrue(auditTask != null && !auditTask.isDone()); + assertEquals("REPLICAS_CHECK_TIME", 0, replicasCheckTime.getSuccessCount()); + + canRun.set(true); + + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(auditor.auditTask.equals(auditTask) + && auditor.auditTask != null && !auditor.auditTask.isDone()); + // wrong successCount is > 0, right successCount is = 0 + assertEquals("REPLICAS_CHECK_TIME", 0, replicasCheckTime.getSuccessCount()); + + auditor.close(); + } + + static class TestAuditor extends Auditor { + + final AtomicReference latchRef = new AtomicReference(new CountDownLatch(1)); + + public TestAuditor(String bookieIdentifier, ServerConfiguration conf, BookKeeper bkc, boolean ownBkc, + StatsLogger statsLogger, AtomicBoolean exceptedRun) throws UnavailableException { + super(bookieIdentifier, conf, bkc, ownBkc, statsLogger); + renewAuditorTestWrapperTask(exceptedRun); + } + + public TestAuditor(String bookieIdentifier, ServerConfiguration conf, BookKeeper bkc, boolean ownBkc, + BookKeeperAdmin bkadmin, boolean ownadmin, StatsLogger statsLogger, + AtomicBoolean exceptedRun) throws UnavailableException { + super(bookieIdentifier, conf, bkc, ownBkc, bkadmin, ownadmin, statsLogger); + renewAuditorTestWrapperTask(exceptedRun); + } + + public TestAuditor(final String bookieIdentifier, ServerConfiguration conf, StatsLogger statsLogger, + AtomicBoolean exceptedRun) + throws UnavailableException { + super(bookieIdentifier, conf, statsLogger); + renewAuditorTestWrapperTask(exceptedRun); + } + + private void renewAuditorTestWrapperTask(AtomicBoolean exceptedRun) { + super.auditorCheckAllLedgersTask = + new AuditorTestWrapperTask(super.auditorCheckAllLedgersTask, latchRef, exceptedRun); + super.auditorPlacementPolicyCheckTask = + new AuditorTestWrapperTask(super.auditorPlacementPolicyCheckTask, latchRef, exceptedRun); + super.auditorReplicasCheckTask = + new AuditorTestWrapperTask(super.auditorReplicasCheckTask, latchRef, exceptedRun); + } + + CountDownLatch getLatch() { + return latchRef.get(); + } + + void setLatch(CountDownLatch latch) { + latchRef.set(latch); + } + + private static class AuditorTestWrapperTask extends AuditorTask { + private final AuditorTask innerTask; + private final AtomicReference latchRef; + private final AtomicBoolean exceptedRun; + + AuditorTestWrapperTask(AuditorTask innerTask, + AtomicReference latchRef, + AtomicBoolean exceptedRun) { + super(null, null, null, null, null, + null, null); + this.innerTask = innerTask; + this.latchRef = latchRef; + this.exceptedRun = exceptedRun; + } + + @Override + protected void runTask() { + if (exceptedRun == null || exceptedRun.get()) { + innerTask.runTask(); + latchRef.get().countDown(); + } + } + + @Override + public void shutdown() { + innerTask.shutdown(); + } + } + } + private BookieId replaceBookieWithWriteFailingBookie(LedgerHandle lh) throws Exception { int bookieIdx = -1; Long entryId = lh.getLedgerMetadata().getAllEnsembles().firstKey(); diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java new file mode 100644 index 0000000000000..1bafb8589d91a --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import java.util.LinkedList; +import java.util.List; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link AuditorPlacementPolicyCheckTask}. + */ +public class AuditorPlacementPolicyCheckTaskTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorPlacementPolicyCheckTaskTest.class); + + private BookKeeperAdmin admin; + private LedgerManager ledgerManager; + private LedgerUnderreplicationManager ledgerUnderreplicationManager; + + public AuditorPlacementPolicyCheckTaskTest() throws Exception { + super(3); + baseConf.setPageLimit(1); + baseConf.setAutoRecoveryDaemonEnabled(false); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + baseClientConf.setMetadataServiceUri( + metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + final BookKeeper bookKeeper = new BookKeeper(baseClientConf); + admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf)); + LedgerManagerFactory ledgerManagerFactory = bookKeeper.getLedgerManagerFactory(); + ledgerManager = ledgerManagerFactory.newLedgerManager(); + ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + if (ledgerManager != null) { + ledgerManager.close(); + } + if (ledgerUnderreplicationManager != null) { + ledgerUnderreplicationManager.close(); + } + super.tearDown(); + } + + @Test + public void testPlacementPolicyCheck() throws BKException, InterruptedException { + + // 1. create ledgers + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + // 2. init auditorPlacementPolicyCheckTask + final TestStatsProvider statsProvider = new TestStatsProvider(); + final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + final AuditorStats auditorStats = new AuditorStats(statsLogger); + + AuditorPlacementPolicyCheckTask auditorPlacementPolicyCheckTask = new AuditorPlacementPolicyCheckTask( + baseConf, auditorStats, admin, ledgerManager, + ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false)); + + // 3. placementPolicyCheck + auditorPlacementPolicyCheckTask.runTask(); + + // 4. verify + assertEquals("PLACEMENT_POLICY_CHECK_TIME", 1, ((TestStatsProvider.TestOpStatsLogger) + statsLogger.getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME)).getSuccessCount()); + assertEquals("numOfClosedLedgersAuditedInPlacementPolicyCheck", + numLedgers, + auditorPlacementPolicyCheckTask.getNumOfClosedLedgersAuditedInPlacementPolicyCheck().get()); + assertEquals("numOfLedgersFoundNotAdheringInPlacementPolicyCheck", + numLedgers, + auditorPlacementPolicyCheckTask.getNumOfLedgersFoundNotAdheringInPlacementPolicyCheck().get()); + } + +} \ No newline at end of file diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java new file mode 100644 index 0000000000000..159a4e88a33bd --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java @@ -0,0 +1,861 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicyImpl.REPP_DNS_RESOLVER_CLASS; +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import java.net.URI; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.ZoneawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.discover.BookieServiceInfo; +import org.apache.bookkeeper.discover.RegistrationManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataBookieDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.exceptions.MetadataException; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.replication.AuditorPeriodicCheckTest.TestAuditor; +import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException; +import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; +import org.apache.bookkeeper.stats.Gauge; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; +import org.apache.bookkeeper.util.StaticDNSResolver; +import org.apache.commons.lang3.mutable.MutableObject; +import org.apache.zookeeper.KeeperException; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests the logic of Auditor's PlacementPolicyCheck. + */ +public class AuditorPlacementPolicyCheckTest extends BookKeeperClusterTestCase { + private MetadataBookieDriver driver; + + public AuditorPlacementPolicyCheckTest() throws Exception { + super(1); + baseConf.setPageLimit(1); // to make it easy to push ledger out of cache + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + StaticDNSResolver.reset(); + + URI uri = URI.create(confByIndex(0).getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver = MetadataDrivers.getBookieDriver(uri); + ServerConfiguration serverConfiguration = new ServerConfiguration(confByIndex(0)); + serverConfiguration.setMetadataServiceUri( + serverConfiguration.getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + if (null != driver) { + driver.close(); + } + super.tearDown(); + } + + @Test + public void testPlacementPolicyCheckWithBookiesFromDifferentRacks() throws Exception { + int numOfBookies = 5; + List bookieAddresses = new ArrayList<>(); + BookieSocketAddress bookieAddress; + RegistrationManager regManager = driver.createRegistrationManager(); + // all the numOfBookies (5) are going to be in different racks + for (int i = 0; i < numOfBookies; i++) { + bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181); + StaticDNSResolver.addNodeToRack(bookieAddress.getHostName(), "/rack" + (i)); + bookieAddresses.add(bookieAddress.toBookieId()); + regManager.registerBookie(bookieAddress.toBookieId(), false, BookieServiceInfo.EMPTY); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 4; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 4; + Collections.shuffle(bookieAddresses); + + // closed ledger + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + + Collections.shuffle(bookieAddresses); + ensembleSize = 4; + // closed ledger with multiple segments + initMeta = LedgerMetadataBuilder.create() + .withId(2L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 4)) + .newEnsembleEntry(20L, bookieAddresses.subList(1, 5)) + .newEnsembleEntry(60L, bookieAddresses.subList(0, 4)) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(2L, initMeta).get(); + + Collections.shuffle(bookieAddresses); + // non-closed ledger + initMeta = LedgerMetadataBuilder.create() + .withId(3L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 4)) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(3L, initMeta).get(); + + Collections.shuffle(bookieAddresses); + // non-closed ledger with multiple segments + initMeta = LedgerMetadataBuilder.create() + .withId(4L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 4)) + .newEnsembleEntry(20L, bookieAddresses.subList(1, 5)) + .newEnsembleEntry(60L, bookieAddresses.subList(0, 4)) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(4L, initMeta).get(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + /* + * since all of the bookies are in different racks, there shouldn't be any ledger not adhering + * to placement policy. + */ + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", 0, + ledgersNotAdheringToPlacementPolicyGuage.getSample()); + /* + * since all of the bookies are in different racks, there shouldn't be any ledger softly adhering + * to placement policy. + */ + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", 0, + ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicy() throws Exception { + int numOfBookies = 5; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 3; + + /* + * this closed ledger doesn't adhere to placement policy because there are only + * 3 racks, and the ensembleSize is 5. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + /* + * this is non-closed ledger, so it shouldn't count as ledger not + * adhering to placement policy + */ + initMeta = LedgerMetadataBuilder.create() + .withId(2L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(2L, initMeta).get(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndNotMarkToUnderreplication() + throws Exception { + int numOfBookies = 5; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 3; + + /* + * this closed ledger doesn't adhere to placement policy because there are only + * 3 racks, and the ensembleSize is 5. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager(); + long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate(); + assertEquals(unnderReplicateLedgerId, -1); + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPlacementPolicyAndMarkToUnderreplication() + throws Exception { + int numOfBookies = 5; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack2"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 3; + + /* + * this closed ledger doesn't adhere to placement policy because there are only + * 3 racks, and the ensembleSize is 5. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager(); + long unnderReplicateLedgerId = underreplicationManager.pollLedgerToRereplicate(); + assertEquals(unnderReplicateLedgerId, 1L); + } + + @Test + public void testPlacementPolicyCheckForURLedgersElapsedRecoveryGracePeriod() throws Exception { + testPlacementPolicyCheckWithURLedgers(true); + } + + @Test + public void testPlacementPolicyCheckForURLedgersNotElapsedRecoveryGracePeriod() throws Exception { + testPlacementPolicyCheckWithURLedgers(false); + } + + public void testPlacementPolicyCheckWithURLedgers(boolean timeElapsed) throws Exception { + int numOfBookies = 4; + /* + * in timeElapsed=true scenario, set some low value, otherwise set some + * highValue. + */ + int underreplicatedLedgerRecoveryGracePeriod = timeElapsed ? 1 : 1000; + int numOfURLedgersElapsedRecoveryGracePeriod = 0; + List bookieAddresses = new ArrayList(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + LedgerUnderreplicationManager underreplicationManager = mFactory.newLedgerUnderreplicationManager(); + int ensembleSize = 4; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + + long ledgerId1 = 1L; + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(ledgerId1) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(ledgerId1, initMeta).get(); + underreplicationManager.markLedgerUnderreplicated(ledgerId1, bookieAddresses.get(0).toString()); + if (timeElapsed) { + numOfURLedgersElapsedRecoveryGracePeriod++; + } + + /* + * this is non-closed ledger, it should also be reported as + * URLedgersElapsedRecoveryGracePeriod + */ + ensembleSize = 3; + long ledgerId2 = 21234561L; + initMeta = LedgerMetadataBuilder.create() + .withId(ledgerId2) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, + Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2))) + .newEnsembleEntry(100L, + Arrays.asList(bookieAddresses.get(3), bookieAddresses.get(1), bookieAddresses.get(2))) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(ledgerId2, initMeta).get(); + underreplicationManager.markLedgerUnderreplicated(ledgerId2, bookieAddresses.get(0).toString()); + if (timeElapsed) { + numOfURLedgersElapsedRecoveryGracePeriod++; + } + + /* + * this ledger is not marked underreplicated. + */ + long ledgerId3 = 31234561L; + initMeta = LedgerMetadataBuilder.create() + .withId(ledgerId3) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, + Arrays.asList(bookieAddresses.get(1), bookieAddresses.get(2), bookieAddresses.get(3))) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(ledgerId3, initMeta).get(); + + if (timeElapsed) { + /* + * in timeelapsed scenario, by waiting for + * underreplicatedLedgerRecoveryGracePeriod, recovery time must be + * elapsed. + */ + Thread.sleep((underreplicatedLedgerRecoveryGracePeriod + 1) * 1000); + } else { + /* + * in timeElapsed=false scenario, since + * underreplicatedLedgerRecoveryGracePeriod is set to some high + * value, there is no value in waiting. So just wait for some time + * and make sure urledgers are not reported as recoverytime elapsed + * urledgers. + */ + Thread.sleep(5000); + } + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setUnderreplicatedLedgerRecoveryGracePeriod(underreplicatedLedgerRecoveryGracePeriod); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge underreplicatedLedgersElapsedRecoveryGracePeriodGuage = statsLogger + .getGauge(ReplicationStats.NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD); + assertEquals("NUM_UNDERREPLICATED_LEDGERS_ELAPSED_RECOVERY_GRACE_PERIOD guage value", + numOfURLedgersElapsedRecoveryGracePeriod, + underreplicatedLedgersElapsedRecoveryGracePeriodGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + @Test + public void testPlacementPolicyCheckWithLedgersNotAdheringToPolicyWithMultipleSegments() throws Exception { + int numOfBookies = 7; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList<>(); + RegistrationManager regManager = driver.createRegistrationManager(); + for (int i = 0; i < numOfBookies; i++) { + BookieId bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + + // only three racks + StaticDNSResolver.addNodeToRack("98.98.98.0", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.1", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.2", "/rack3"); + StaticDNSResolver.addNodeToRack("98.98.98.3", "/rack4"); + StaticDNSResolver.addNodeToRack("98.98.98.4", "/rack1"); + StaticDNSResolver.addNodeToRack("98.98.98.5", "/rack2"); + StaticDNSResolver.addNodeToRack("98.98.98.6", "/rack3"); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 5; + int ackQuorumSize = 2; + int minNumRacksPerWriteQuorumConfValue = 4; + + /* + * this closed ledger in each writeQuorumSize (5), there would be + * atleast minNumRacksPerWriteQuorumConfValue (4) racks. So it wont be + * counted as ledgers not adhering to placement policy. + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 5)) + .newEnsembleEntry(20L, bookieAddresses.subList(1, 6)) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + + /* + * for the second segment bookies are from /rack1, /rack2 and /rack3, + * which is < minNumRacksPerWriteQuorumConfValue (4). So it is not + * adhering to placement policy. + * + * also for the third segment are from /rack1, /rack2 and /rack3, which + * is < minNumRacksPerWriteQuorumConfValue (4). So it is not adhering to + * placement policy. + * + * Though there are multiple segments are not adhering to placement + * policy, it should be counted as single ledger. + */ + initMeta = LedgerMetadataBuilder.create() + .withId(2L) + .withEnsembleSize(ensembleSize) + .withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize) + .newEnsembleEntry(0L, bookieAddresses.subList(0, 5)) + .newEnsembleEntry(20L, + Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2), + bookieAddresses.get(4), bookieAddresses.get(5))) + .newEnsembleEntry(40L, + Arrays.asList(bookieAddresses.get(0), bookieAddresses.get(1), bookieAddresses.get(2), + bookieAddresses.get(4), bookieAddresses.get(6))) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(2L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + setServerConfigPropertiesForRackPlacement(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY gauge value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY gauge value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + @Test + public void testZoneawarePlacementPolicyCheck() throws Exception { + int numOfBookies = 6; + int numOfLedgersNotAdheringToPlacementPolicy = 0; + int numOfLedgersSoftlyAdheringToPlacementPolicy = 0; + List bookieAddresses = new ArrayList(); + RegistrationManager regManager = driver.createRegistrationManager(); + /* + * 6 bookies - 3 zones and 2 uds + */ + for (int i = 0; i < numOfBookies; i++) { + BookieSocketAddress bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181); + bookieAddresses.add(bookieAddress.toBookieId()); + regManager.registerBookie(bookieAddress.toBookieId(), false, BookieServiceInfo.EMPTY); + String zone = "/zone" + (i % 3); + String upgradeDomain = "/ud" + (i % 2); + String networkLocation = zone + upgradeDomain; + StaticDNSResolver.addNodeToRack(bookieAddress.getHostName(), networkLocation); + } + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setDesiredNumZonesPerWriteQuorum(3); + servConf.setMinNumZonesPerWriteQuorum(2); + setServerConfigPropertiesForZonePlacement(servConf); + + /* + * this closed ledger adheres to ZoneAwarePlacementPolicy, since + * ensemble is spread across 3 zones and 2 UDs + */ + LedgerMetadata initMeta = LedgerMetadataBuilder.create() + .withId(1L) + .withEnsembleSize(6) + .withWriteQuorumSize(6) + .withAckQuorumSize(2) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(1L, initMeta).get(); + + /* + * this is non-closed ledger, so though ensemble is not adhering to + * placement policy (since ensemble is not multiple of writeQuorum), + * this shouldn't be reported + */ + initMeta = LedgerMetadataBuilder.create() + .withId(2L) + .withEnsembleSize(6) + .withWriteQuorumSize(5) + .withAckQuorumSize(2) + .newEnsembleEntry(0L, bookieAddresses) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(2L, initMeta).get(); + + /* + * this is closed ledger, since ensemble is not multiple of writeQuorum, + * this ledger is not adhering to placement policy. + */ + initMeta = LedgerMetadataBuilder.create() + .withId(3L) + .withEnsembleSize(6) + .withWriteQuorumSize(5) + .withAckQuorumSize(2) + .newEnsembleEntry(0L, bookieAddresses) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(3L, initMeta).get(); + numOfLedgersNotAdheringToPlacementPolicy++; + + /* + * this closed ledger adheres softly to ZoneAwarePlacementPolicy, since + * ensemble/writeQuorum of size 4 has spread across just + * minNumZonesPerWriteQuorum (2). + */ + List newEnsemble = new ArrayList(); + newEnsemble.add(bookieAddresses.get(0)); + newEnsemble.add(bookieAddresses.get(1)); + newEnsemble.add(bookieAddresses.get(3)); + newEnsemble.add(bookieAddresses.get(4)); + initMeta = LedgerMetadataBuilder.create() + .withId(4L) + .withEnsembleSize(4) + .withWriteQuorumSize(4) + .withAckQuorumSize(2) + .newEnsembleEntry(0L, newEnsemble) + .withClosedState() + .withLastEntryId(100) + .withLength(10000) + .withDigestType(DigestType.DUMMY) + .withPassword(new byte[0]) + .build(); + lm.createLedgerMetadata(4L, initMeta).get(); + numOfLedgersSoftlyAdheringToPlacementPolicy++; + + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersNotAdheringToPlacementPolicy, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + numOfLedgersSoftlyAdheringToPlacementPolicy, + ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + regManager.close(); + } + } + + private void setServerConfigPropertiesForRackPlacement(ServerConfiguration servConf) { + setServerConfigProperties(servConf, RackawareEnsemblePlacementPolicy.class.getName()); + } + + private void setServerConfigPropertiesForZonePlacement(ServerConfiguration servConf) { + setServerConfigProperties(servConf, ZoneawareEnsemblePlacementPolicy.class.getName()); + } + + private void setServerConfigProperties(ServerConfiguration servConf, String ensemblePlacementPolicyClass) { + servConf.setProperty(REPP_DNS_RESOLVER_CLASS, StaticDNSResolver.class.getName()); + servConf.setProperty(ClientConfiguration.ENSEMBLE_PLACEMENT_POLICY, ensemblePlacementPolicyClass); + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + servConf.setAuditorPeriodicReplicasCheckInterval(0); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); + } + + private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf, + MutableObject auditorRef) throws MetadataException, CompatibilityException, KeeperException, + InterruptedException, UnavailableException, UnknownHostException { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger placementPolicyCheckStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, + statsLogger, null); + auditorRef.setValue(auditor); + CountDownLatch latch = auditor.getLatch(); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, placementPolicyCheckStatsLogger.getSuccessCount()); + urm.setPlacementPolicyCheckCTime(-1); + auditor.start(); + /* + * since placementPolicyCheckCTime is set to -1, placementPolicyCheck should be + * scheduled to run with no initialdelay + */ + assertTrue("placementPolicyCheck should have executed", latch.await(20, TimeUnit.SECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, placementPolicyCheckStatsLogger.getSuccessCount()); + return statsLogger; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java new file mode 100644 index 0000000000000..21dd2807b75d3 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import java.util.LinkedList; +import java.util.List; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Unit test {@link AuditorReplicasCheckTask}. + */ +public class AuditorReplicasCheckTaskTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(AuditorReplicasCheckTaskTest.class); + + private BookKeeperAdmin admin; + private LedgerManager ledgerManager; + private LedgerUnderreplicationManager ledgerUnderreplicationManager; + + public AuditorReplicasCheckTaskTest() throws Exception { + super(3); + baseConf.setPageLimit(1); + baseConf.setAutoRecoveryDaemonEnabled(false); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + baseClientConf.setMetadataServiceUri( + metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + final BookKeeper bookKeeper = new BookKeeper(baseClientConf); + admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new ClientConfiguration(baseClientConf)); + LedgerManagerFactory ledgerManagerFactory = bookKeeper.getLedgerManagerFactory(); + ledgerManager = ledgerManagerFactory.newLedgerManager(); + ledgerUnderreplicationManager = ledgerManagerFactory.newLedgerUnderreplicationManager(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + if (ledgerManager != null) { + ledgerManager.close(); + } + if (ledgerUnderreplicationManager != null) { + ledgerUnderreplicationManager.close(); + } + if (admin != null) { + admin.close(); + } + super.tearDown(); + } + + @Test + public void testReplicasCheck() throws BKException, InterruptedException { + + // 1. create ledgers + final int numLedgers = 10; + List ids = new LinkedList(); + for (int i = 0; i < numLedgers; i++) { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, "passwd".getBytes()); + ids.add(lh.getId()); + for (int j = 0; j < 2; j++) { + lh.addEntry("testdata".getBytes()); + } + lh.close(); + } + + // 2. init auditorReplicasCheckTask + final TestStatsProvider statsProvider = new TestStatsProvider(); + final TestStatsProvider.TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + final AuditorStats auditorStats = new AuditorStats(statsLogger); + AuditorReplicasCheckTask auditorReplicasCheckTask = new AuditorReplicasCheckTask( + baseConf, auditorStats, admin, ledgerManager, + ledgerUnderreplicationManager, null, (flag, throwable) -> flag.set(false)); + + // 3. replicasCheck + auditorReplicasCheckTask.runTask(); + + // 4. verify + assertEquals("REPLICAS_CHECK_TIME", 1, ((TestStatsProvider.TestOpStatsLogger) + statsLogger.getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME)).getSuccessCount()); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java new file mode 100644 index 0000000000000..a4d6d86deced2 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java @@ -0,0 +1,937 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import java.net.URI; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.LedgerMetadataBuilder; +import org.apache.bookkeeper.client.api.DigestType; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.discover.BookieServiceInfo; +import org.apache.bookkeeper.discover.RegistrationManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataBookieDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.exceptions.MetadataException; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.replication.AuditorPeriodicCheckTest.TestAuditor; +import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException; +import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; +import org.apache.bookkeeper.stats.Gauge; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; +import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; +import org.apache.bookkeeper.util.AvailabilityOfEntriesOfLedger; +import org.apache.bookkeeper.util.StaticDNSResolver; +import org.apache.commons.collections4.map.MultiKeyMap; +import org.apache.commons.lang3.mutable.MutableObject; +import org.apache.zookeeper.KeeperException; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests the logic of Auditor's ReplicasCheck. + */ +public class AuditorReplicasCheckTest extends BookKeeperClusterTestCase { + private MetadataBookieDriver driver; + private RegistrationManager regManager; + + public AuditorReplicasCheckTest() throws Exception { + super(1); + baseConf.setPageLimit(1); // to make it easy to push ledger out of cache + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + StaticDNSResolver.reset(); + + URI uri = URI.create(confByIndex(0).getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver = MetadataDrivers.getBookieDriver(uri); + ServerConfiguration serverConfiguration = new ServerConfiguration(confByIndex(0)); + serverConfiguration.setMetadataServiceUri( + serverConfiguration.getMetadataServiceUri().replaceAll("zk://", "metadata-store:") + .replaceAll("/ledgers", "")); + driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE); + regManager = driver.createRegistrationManager(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + if (null != regManager) { + regManager.close(); + } + if (null != driver) { + driver.close(); + } + super.tearDown(); + } + + private class TestBookKeeperAdmin extends BookKeeperAdmin { + + private final MultiKeyMap returnAvailabilityOfEntriesOfLedger; + private final MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger; + + public TestBookKeeperAdmin(BookKeeper bkc, StatsLogger statsLogger, + MultiKeyMap returnAvailabilityOfEntriesOfLedger, + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger) { + super(bkc, statsLogger, baseClientConf); + this.returnAvailabilityOfEntriesOfLedger = returnAvailabilityOfEntriesOfLedger; + this.errorReturnValueForGetAvailabilityOfEntriesOfLedger = + errorReturnValueForGetAvailabilityOfEntriesOfLedger; + } + + @Override + public CompletableFuture asyncGetListOfEntriesOfLedger( + BookieId address, long ledgerId) { + CompletableFuture futureResult = + new CompletableFuture(); + Integer errorReturnValue = errorReturnValueForGetAvailabilityOfEntriesOfLedger.get(address.toString(), + Long.toString(ledgerId)); + if (errorReturnValue != null) { + futureResult.completeExceptionally(BKException.create(errorReturnValue).fillInStackTrace()); + } else { + AvailabilityOfEntriesOfLedger availabilityOfEntriesOfLedger = returnAvailabilityOfEntriesOfLedger + .get(address.toString(), Long.toString(ledgerId)); + futureResult.complete(availabilityOfEntriesOfLedger); + } + return futureResult; + } + } + + private TestStatsLogger startAuditorAndWaitForReplicasCheck(ServerConfiguration servConf, + MutableObject auditorRef, + MultiKeyMap expectedReturnAvailabilityOfEntriesOfLedger, + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger) + throws MetadataException, CompatibilityException, KeeperException, InterruptedException, + UnavailableException, UnknownHostException { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestOpStatsLogger replicasCheckStatsLogger = (TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.REPLICAS_CHECK_TIME); + + final TestAuditor auditor = new TestAuditor(BookieImpl.getBookieId(servConf).toString(), servConf, bkc, true, + new TestBookKeeperAdmin(bkc, statsLogger, expectedReturnAvailabilityOfEntriesOfLedger, + errorReturnValueForGetAvailabilityOfEntriesOfLedger), + true, statsLogger, null); + auditorRef.setValue(auditor); + CountDownLatch latch = auditor.getLatch(); + assertEquals("REPLICAS_CHECK_TIME SuccessCount", 0, replicasCheckStatsLogger.getSuccessCount()); + urm.setReplicasCheckCTime(-1); + auditor.start(); + /* + * since replicasCheckCTime is set to -1, replicasCheck should be + * scheduled to run with no initialdelay + */ + assertTrue("replicasCheck should have executed", latch.await(20, TimeUnit.SECONDS)); + for (int i = 0; i < 200; i++) { + Thread.sleep(100); + if (replicasCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("REPLICAS_CHECK_TIME SuccessCount", 1, replicasCheckStatsLogger.getSuccessCount()); + return statsLogger; + } + + private void setServerConfigProperties(ServerConfiguration servConf) { + servConf.setAuditorPeriodicCheckInterval(0); + servConf.setAuditorPeriodicBookieCheckInterval(0); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(0); + servConf.setAuditorPeriodicReplicasCheckInterval(1000); + } + + List addAndRegisterBookies(int numOfBookies) + throws BookieException { + BookieId bookieAddress; + List bookieAddresses = new ArrayList(); + for (int i = 0; i < numOfBookies; i++) { + bookieAddress = new BookieSocketAddress("98.98.98." + i, 2181).toBookieId(); + bookieAddresses.add(bookieAddress); + regManager.registerBookie(bookieAddress, false, BookieServiceInfo.EMPTY); + } + return bookieAddresses; + } + + private void createClosedLedgerMetadata(LedgerManager lm, long ledgerId, int ensembleSize, int writeQuorumSize, + int ackQuorumSize, Map> segmentEnsembles, long lastEntryId, int length, + DigestType digestType, byte[] password) throws InterruptedException, ExecutionException { + LedgerMetadataBuilder ledgerMetadataBuilder = LedgerMetadataBuilder.create(); + ledgerMetadataBuilder.withId(ledgerId).withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize).withClosedState().withLastEntryId(lastEntryId).withLength(length) + .withDigestType(digestType).withPassword(password); + for (Map.Entry> mapEntry : segmentEnsembles.entrySet()) { + ledgerMetadataBuilder.newEnsembleEntry(mapEntry.getKey(), mapEntry.getValue()); + } + LedgerMetadata initMeta = ledgerMetadataBuilder.build(); + lm.createLedgerMetadata(ledgerId, initMeta).get(); + } + + private void createNonClosedLedgerMetadata(LedgerManager lm, long ledgerId, int ensembleSize, int writeQuorumSize, + int ackQuorumSize, Map> segmentEnsembles, DigestType digestType, + byte[] password) throws InterruptedException, ExecutionException { + LedgerMetadataBuilder ledgerMetadataBuilder = LedgerMetadataBuilder.create(); + ledgerMetadataBuilder.withId(ledgerId).withEnsembleSize(ensembleSize).withWriteQuorumSize(writeQuorumSize) + .withAckQuorumSize(ackQuorumSize).withDigestType(digestType).withPassword(password); + for (Map.Entry> mapEntry : segmentEnsembles.entrySet()) { + ledgerMetadataBuilder.newEnsembleEntry(mapEntry.getKey(), mapEntry.getValue()); + } + LedgerMetadata initMeta = ledgerMetadataBuilder.build(); + lm.createLedgerMetadata(ledgerId, initMeta).get(); + } + + private void runTestScenario(MultiKeyMap returnAvailabilityOfEntriesOfLedger, + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger, + int expectedNumLedgersFoundHavingNoReplicaOfAnEntry, + int expectedNumLedgersHavingLessThanAQReplicasOfAnEntry, + int expectedNumLedgersHavingLessThanWQReplicasOfAnEntry) throws Exception { + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + setServerConfigProperties(servConf); + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForReplicasCheck(servConf, auditorRef, + returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger); + checkReplicasCheckStats(statsLogger, expectedNumLedgersFoundHavingNoReplicaOfAnEntry, + expectedNumLedgersHavingLessThanAQReplicasOfAnEntry, + expectedNumLedgersHavingLessThanWQReplicasOfAnEntry); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + } + } + + private void checkReplicasCheckStats(TestStatsLogger statsLogger, + int expectedNumLedgersFoundHavingNoReplicaOfAnEntry, + int expectedNumLedgersHavingLessThanAQReplicasOfAnEntry, + int expectedNumLedgersHavingLessThanWQReplicasOfAnEntry) { + Gauge numLedgersFoundHavingNoReplicaOfAnEntryGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY); + Gauge numLedgersHavingLessThanAQReplicasOfAnEntryGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_AQ_REPLICAS_OF_AN_ENTRY); + Gauge numLedgersHavingLessThanWQReplicasOfAnEntryGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY); + + assertEquals("NUM_LEDGERS_HAVING_NO_REPLICA_OF_AN_ENTRY guage value", + expectedNumLedgersFoundHavingNoReplicaOfAnEntry, + numLedgersFoundHavingNoReplicaOfAnEntryGuage.getSample()); + assertEquals("NUM_LEDGERS_HAVING_LESS_THAN_AQ_REPLICAS_OF_AN_ENTRY guage value", + expectedNumLedgersHavingLessThanAQReplicasOfAnEntry, + numLedgersHavingLessThanAQReplicasOfAnEntryGuage.getSample()); + assertEquals("NUM_LEDGERS_HAVING_LESS_THAN_WQ_REPLICAS_OF_AN_ENTRY guage value", + expectedNumLedgersHavingLessThanWQReplicasOfAnEntry, + numLedgersHavingLessThanWQReplicasOfAnEntryGuage.getSample()); + } + + /* + * For all the ledgers and for all the bookies, + * asyncGetListOfEntriesOfLedger would return + * BookieHandleNotAvailableException, so these ledgers wouldn't be counted + * against expectedNumLedgersFoundHavingNoReplicaOfAnEntry / + * LessThanAQReplicasOfAnEntry / LessThanWQReplicasOfAnEntry. + */ + @Test + public void testReplicasCheckForBookieHandleNotAvailable() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 4; + int ackQuorumSize = 2; + long lastEntryId = 100; + int length = 10000; + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + /* + * closed ledger + * + * for this ledger, for all the bookies we are setting + * errorReturnValueForGetAvailabilityOfEntriesOfLedger to + * BookieHandleNotAvailableException so asyncGetListOfEntriesOfLedger will + * return BookieHandleNotAvailableException. + */ + Map> segmentEnsembles = new LinkedHashMap>(); + segmentEnsembles.put(0L, bookieAddresses); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + } + + ensembleSize = 4; + /* + * closed ledger with multiple segments + * + * for this ledger, for all the bookies we are setting + * errorReturnValueForGetAvailabilityOfEntriesOfLedger to + * BookieHandleNotAvailableException so asyncGetListOfEntriesOfLedger will + * return BookieHandleNotAvailableException. + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(20L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(60L, bookieAddresses.subList(0, 4)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + } + + /* + * non-closed ledger + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + ledgerId = 3L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + } + + /* + * non-closed ledger with multiple segments + * + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(20L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(60L, bookieAddresses.subList(0, 4)); + ledgerId = 4L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + } + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0, 0, + 0); + } + + /* + * In this testscenario all the ledgers have a missing entry. So all closed + * ledgers should be counted towards + * numLedgersFoundHavingNoReplicaOfAnEntry. + */ + @Test + public void testReplicasCheckForLedgersFoundHavingNoReplica() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + int ensembleSize = 5; + int writeQuorumSize = 4; + int ackQuorumSize = 2; + long lastEntryId = 100; + int length = 10000; + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + int numLedgersFoundHavingNoReplicaOfAnEntry = 0; + + /* + * closed ledger + * + * for this ledger we are setting returnAvailabilityOfEntriesOfLedger to + * Empty one for all of the bookies, so this ledger would be counted in + * ledgersFoundHavingNoReplicaOfAnEntry . + */ + Map> segmentEnsembles = new LinkedHashMap>(); + segmentEnsembles.put(0L, bookieAddresses); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + } + numLedgersFoundHavingNoReplicaOfAnEntry++; + + ensembleSize = 4; + /* + * closed ledger with multiple segments + * + * for this ledger we are setting + * errorReturnValueForGetAvailabilityOfEntriesOfLedger to + * NoSuchLedgerExistsException. This is equivalent to + * EMPTY_AVAILABILITYOFENTRIESOFLEDGER. So this ledger would be counted + * in ledgersFoundHavingNoReplicaOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(20L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(60L, bookieAddresses.subList(0, 4)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), + Long.toString(ledgerId), BKException.Code.NoSuchLedgerExistsException); + } + numLedgersFoundHavingNoReplicaOfAnEntry++; + + /* + * non-closed ledger + * + * since this is non-closed ledger, it should not be counted in + * ledgersFoundHavingNoReplicaOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + ledgerId = 3L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + } + + ensembleSize = 3; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 1; + length = 1000; + /* + * closed ledger + * + * for this ledger we are setting returnAvailabilityOfEntriesOfLedger to + * just {0l} for all of the bookies and entry 1l is missing for all of + * the bookies, so this ledger would be counted in + * ledgersFoundHavingNoReplicaOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 3)); + ledgerId = 4L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + for (BookieId bookieSocketAddress : bookieAddresses) { + returnAvailabilityOfEntriesOfLedger.put(bookieSocketAddress.toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0L })); + } + numLedgersFoundHavingNoReplicaOfAnEntry++; + + /* + * For this closed ledger, entry 1 is missing. So it should be counted + * towards numLedgersFoundHavingNoReplicaOfAnEntry. + */ + ensembleSize = 4; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 3; + length = 10000; + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + ledgerId = 5L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 })); + numLedgersFoundHavingNoReplicaOfAnEntry++; + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, + numLedgersFoundHavingNoReplicaOfAnEntry, 0, 0); + } + + /* + * In this testscenario all the ledgers have an entry with less than AQ + * number of copies. So all closed ledgers should be counted towards + * numLedgersFoundHavingLessThanAQReplicasOfAnEntry. + */ + @Test + public void testReplicasCheckForLedgersFoundHavingLessThanAQReplicasOfAnEntry() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + int numLedgersFoundHavingLessThanAQReplicasOfAnEntry = 0; + + /* + * closed ledger + * + * for this ledger there is only one copy of entry 2, so this ledger + * would be counted towards + * ledgersFoundHavingLessThanAQReplicasOfAnEntry. + */ + Map> segmentEnsembles = new LinkedHashMap>(); + int ensembleSize = 4; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + long lastEntryId = 3; + int length = 10000; + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2, 3 })); + numLedgersFoundHavingLessThanAQReplicasOfAnEntry++; + + /* + * closed ledger with multiple segments. + * + * for this ledger there is only one copy of entry 2, so this ledger + * would be counted towards + * ledgersFoundHavingLessThanAQReplicasOfAnEntry. + * + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] {})); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 3 })); + numLedgersFoundHavingLessThanAQReplicasOfAnEntry++; + + /* + * closed ledger with multiple segments + * + * for this ledger entry 2 is overrreplicated, but it has only one copy + * in the set of bookies it is supposed to be. So it should be counted + * towards ledgersFoundHavingLessThanAQReplicasOfAnEntry. + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 3L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 3 })); + numLedgersFoundHavingLessThanAQReplicasOfAnEntry++; + + /* + * non-closed ledger + * + * since this is non-closed ledger, it should not be counted towards + * ledgersFoundHavingLessThanAQReplicasOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 4L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] {})); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 3 })); + + /* + * this is closed ledger. + * + * For third bookie, asyncGetListOfEntriesOfLedger will fail with + * BookieHandleNotAvailableException, so this should not be counted + * against missing copies of an entry. Other than that, for both entries + * 0 and 1, two copies are missing. Hence this should be counted towards + * numLedgersFoundHavingLessThanAQReplicasOfAnEntry. + */ + ensembleSize = 3; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 1; + length = 1000; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 3)); + ledgerId = 5L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + numLedgersFoundHavingLessThanAQReplicasOfAnEntry++; + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0, + numLedgersFoundHavingLessThanAQReplicasOfAnEntry, 0); + } + + /* + * In this testscenario all the ledgers have an entry with less than WQ + * number of copies but greater than AQ. So all closed ledgers should be + * counted towards numLedgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + @Test + public void testReplicasCheckForLedgersFoundHavingLessThanWQReplicasOfAnEntry() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + int numLedgersFoundHavingLessThanWQReplicasOfAnEntry = 0; + + /* + * closed ledger + * + * for this ledger a copy of entry 3, so this ledger would be counted + * towards ledgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + Map> segmentEnsembles = new LinkedHashMap>(); + int ensembleSize = 4; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + long lastEntryId = 3; + int length = 10000; + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2, 3 })); + numLedgersFoundHavingLessThanWQReplicasOfAnEntry++; + + /* + * closed ledger with multiple segments + * + * for this ledger a copy of entry 0 and entry 2 are missing, so this + * ledger would be counted towards + * ledgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] {})); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 })); + numLedgersFoundHavingLessThanWQReplicasOfAnEntry++; + + /* + * non-closed ledger with multiple segments + * + * since this is non-closed ledger, it should not be counted towards + * ledgersFoundHavingLessThanWQReplicasOfAnEntry + */ + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(2L, bookieAddresses.subList(1, 5)); + ledgerId = 3L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), + Long.toString(ledgerId), BKException.Code.NoSuchLedgerExistsException); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(4).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 2, 3 })); + + /* + * closed ledger. + * + * for this ledger entry 0 is overrreplicated, but a copy is missing in + * the set of bookies it is supposed to be. So it should be counted + * towards ledgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + ensembleSize = 4; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 1; + length = 1000; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + ledgerId = 4L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 3 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0 })); + numLedgersFoundHavingLessThanWQReplicasOfAnEntry++; + + /* + * this is closed ledger. + * + * For third bookie, asyncGetListOfEntriesOfLedger will fail with + * BookieHandleNotAvailableException, so this should not be counted + * against missing copies of an entry. Other than that, for both entries + * 0 and 1, a copy is missing. Hence this should be counted towards + * numLedgersFoundHavingLessThanWQReplicasOfAnEntry. + */ + ensembleSize = 3; + writeQuorumSize = 3; + ackQuorumSize = 2; + lastEntryId = 1; + length = 1000; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 3)); + ledgerId = 5L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + AvailabilityOfEntriesOfLedger.EMPTY_AVAILABILITYOFENTRIESOFLEDGER); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + errorReturnValueForGetAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), + Long.toString(ledgerId), BKException.Code.BookieHandleNotAvailableException); + numLedgersFoundHavingLessThanWQReplicasOfAnEntry++; + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, 0, 0, + numLedgersFoundHavingLessThanWQReplicasOfAnEntry); + } + + /* + * In this testscenario all the ledgers have empty segments. + */ + @Test + public void testReplicasCheckForLedgersWithEmptySegments() throws Exception { + int numOfBookies = 5; + MultiKeyMap returnAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + MultiKeyMap errorReturnValueForGetAvailabilityOfEntriesOfLedger = + new MultiKeyMap(); + List bookieAddresses = addAndRegisterBookies(numOfBookies); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerManager lm = mFactory.newLedgerManager(); + DigestType digestType = DigestType.DUMMY; + byte[] password = new byte[0]; + Collections.shuffle(bookieAddresses); + + int numLedgersFoundHavingNoReplicaOfAnEntry = 0; + int numLedgersFoundHavingLessThanAQReplicasOfAnEntry = 0; + int numLedgersFoundHavingLessThanWQReplicasOfAnEntry = 0; + + /* + * closed ledger. + * + * This closed Ledger has no entry. So it should not be counted towards + * numLedgersFoundHavingNoReplicaOfAnEntry/LessThanAQReplicasOfAnEntry + * /WQReplicasOfAnEntry. + */ + Map> segmentEnsembles = new LinkedHashMap>(); + int ensembleSize = 4; + int writeQuorumSize = 3; + int ackQuorumSize = 2; + long lastEntryId = -1L; + int length = 0; + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + long ledgerId = 1L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + + /* + * closed ledger with multiple segments. + * + * This ledger has empty last segment, but all the entries have + * writeQuorumSize number of copies, So it should not be counted towards + * numLedgersFoundHavingNoReplicaOfAnEntry/LessThanAQReplicasOfAnEntry/ + * WQReplicasOfAnEntry. + */ + lastEntryId = 2; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5)); + ledgerId = 2L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 })); + + /* + * Closed ledger with multiple segments. + * + * Segment0, Segment1, Segment3, Segment5 and Segment6 are empty. + * Entries from entryid 3 are missing. So it should be counted towards + * numLedgersFoundHavingNoReplicaOfAnEntry. + */ + lastEntryId = 5; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(4L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put(4L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5)); + segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(0, 4)); + ledgerId = 3L; + createClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + lastEntryId, length, digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 })); + numLedgersFoundHavingNoReplicaOfAnEntry++; + + /* + * non-closed ledger with multiple segments + * + * since this is non-closed ledger, it should not be counted towards + * ledgersFoundHavingLessThanWQReplicasOfAnEntry + */ + lastEntryId = 2; + segmentEnsembles.clear(); + segmentEnsembles.put(0L, bookieAddresses.subList(0, 4)); + segmentEnsembles.put(0L, bookieAddresses.subList(1, 5)); + segmentEnsembles.put((lastEntryId + 1), bookieAddresses.subList(1, 5)); + ledgerId = 4L; + createNonClosedLedgerMetadata(lm, ledgerId, ensembleSize, writeQuorumSize, ackQuorumSize, segmentEnsembles, + digestType, password); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(0).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(1).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(2).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 0, 1, 2 })); + returnAvailabilityOfEntriesOfLedger.put(bookieAddresses.get(3).toString(), Long.toString(ledgerId), + new AvailabilityOfEntriesOfLedger(new long[] { 1, 2 })); + + runTestScenario(returnAvailabilityOfEntriesOfLedger, errorReturnValueForGetAvailabilityOfEntriesOfLedger, + numLedgersFoundHavingNoReplicaOfAnEntry, numLedgersFoundHavingLessThanAQReplicasOfAnEntry, + numLedgersFoundHavingLessThanWQReplicasOfAnEntry); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java new file mode 100644 index 0000000000000..2c458d635f528 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.meta.MetadataDrivers.runFunctionWithLedgerManagerFactory; +import static org.testng.AssertJUnit.assertEquals; +import com.google.common.util.concurrent.UncheckedExecutionException; +import lombok.Cleanup; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerAuditorManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestCallbacks; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test auditor behaviours during a rolling restart. + */ +public class AuditorRollingRestartTest extends BookKeeperClusterTestCase { + + public AuditorRollingRestartTest() throws Exception { + super(3, 600); + // run the daemon within the bookie + setAutoRecoveryEnabled(true); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + @Override + protected void startBKCluster(String metadataServiceUri) throws Exception { + super.startBKCluster(metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + } + + /** + * Test no auditing during restart if disabled. + */ + @Test + public void testAuditingDuringRollingRestart() throws Exception { + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + runFunctionWithLedgerManagerFactory( + confByIndex(0), + mFactory -> { + try { + testAuditingDuringRollingRestart(mFactory); + } catch (Exception e) { + throw new UncheckedExecutionException(e.getMessage(), e); + } + return null; + } + ); + } + + private void testAuditingDuringRollingRestart(LedgerManagerFactory mFactory) throws Exception { + final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + + LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes()); + for (int i = 0; i < 10; i++) { + lh.asyncAddEntry("foobar".getBytes(), new TestCallbacks.AddCallbackFuture(i), null); + } + lh.addEntry("foobar".getBytes()); + lh.close(); + + assertEquals("shouldn't be anything under replicated", + underReplicationManager.pollLedgerToRereplicate(), -1); + underReplicationManager.disableLedgerReplication(); + + @Cleanup + LedgerAuditorManager lam = mFactory.newLedgerAuditorManager(); + BookieId auditor = lam.getCurrentAuditor(); + ServerConfiguration conf = killBookie(auditor); + Thread.sleep(2000); + startBookie(conf); + Thread.sleep(2000); // give it time to run + assertEquals("shouldn't be anything under replicated", -1, + underReplicationManager.pollLedgerToRereplicate()); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java new file mode 100644 index 0000000000000..db338d1bb4b39 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import org.apache.bookkeeper.auth.AuthCallbacks; +import org.apache.bookkeeper.auth.AuthToken; +import org.apache.bookkeeper.auth.ClientAuthProvider; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.proto.ClientConnectionPeer; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * This test verifies the auditor bookie scenarios from the auth point-of-view. + */ +public class AuthAutoRecoveryTest extends BookKeeperClusterTestCase { + + private static final Logger LOG = LoggerFactory + .getLogger(AuthAutoRecoveryTest.class); + + public static final String TEST_AUTH_PROVIDER_PLUGIN_NAME = "TestAuthProviderPlugin"; + + private static String clientSideRole; + + private static class AuditorClientAuthInterceptorFactory + implements ClientAuthProvider.Factory { + + @Override + public String getPluginName() { + return TEST_AUTH_PROVIDER_PLUGIN_NAME; + } + + @Override + public void init(ClientConfiguration conf) { + clientSideRole = conf.getClientRole(); + } + + @Override + public ClientAuthProvider newProvider(ClientConnectionPeer addr, + final AuthCallbacks.GenericCallback completeCb) { + return new ClientAuthProvider() { + public void init(AuthCallbacks.GenericCallback cb) { + completeCb.operationComplete(BKException.Code.OK, null); + } + + public void process(AuthToken m, AuthCallbacks.GenericCallback cb) { + } + }; + } + } + + protected ServerConfiguration newServerConfiguration() throws Exception { + ServerConfiguration conf = super.newServerConfiguration(); + conf.setClientAuthProviderFactoryClass(AuditorClientAuthInterceptorFactory.class.getName()); + return conf; + } + + public AuthAutoRecoveryTest() { + super(6); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + /* + * test the client role of the auditor + */ + @Test + public void testAuthClientRole() throws Exception { + ServerConfiguration config = confByIndex(0); + assertEquals(AuditorClientAuthInterceptorFactory.class.getName(), config.getClientAuthProviderFactoryClass()); + AutoRecoveryMain main = new AutoRecoveryMain(config); + try { + main.start(); + Thread.sleep(500); + assertTrue("AuditorElector should be running", + main.auditorElector.isRunning()); + assertTrue("Replication worker should be running", + main.replicationWorker.isRunning()); + } finally { + main.shutdown(); + } + assertEquals(ClientConfiguration.CLIENT_ROLE_SYSTEM, clientSideRole); + } + +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java new file mode 100644 index 0000000000000..62416968142b1 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertTrue; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.util.TestUtils; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.Awaitility; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test the AuditorPeer. + */ +@Slf4j +public class AutoRecoveryMainTest extends BookKeeperClusterTestCase { + + public AutoRecoveryMainTest() throws Exception { + super(3); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + /** + * Test the startup of the auditorElector and RW. + */ + @Test + public void testStartup() throws Exception { + log.info("testStartup()"); + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + AutoRecoveryMain main = new AutoRecoveryMain(confByIndex(0)); + try { + main.start(); + Thread.sleep(500); + assertTrue("AuditorElector should be running", + main.auditorElector.isRunning()); + assertTrue("Replication worker should be running", + main.replicationWorker.isRunning()); + } finally { + main.shutdown(); + } + } + + /* + * Test the shutdown of all daemons + */ + @Test + public void testShutdown() throws Exception { + log.info("testShutdown()"); + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + AutoRecoveryMain main = new AutoRecoveryMain(confByIndex(0)); + main.start(); + Thread.sleep(500); + assertTrue("AuditorElector should be running", + main.auditorElector.isRunning()); + assertTrue("Replication worker should be running", + main.replicationWorker.isRunning()); + + main.shutdown(); + assertFalse("AuditorElector should not be running", + main.auditorElector.isRunning()); + assertFalse("Replication worker should not be running", + main.replicationWorker.isRunning()); + } + + /** + * Test that, if an autorecovery looses its ZK connection/session it will + * shutdown. + */ + @Test + public void testAutoRecoverySessionLoss() throws Exception { + log.info("testAutoRecoverySessionLoss()"); + confByIndex(0).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + confByIndex(1).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + confByIndex(2).setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + /* + * initialize three AutoRecovery instances. + */ + AutoRecoveryMain main1 = new AutoRecoveryMain(confByIndex(0)); + AutoRecoveryMain main2 = new AutoRecoveryMain(confByIndex(1)); + AutoRecoveryMain main3 = new AutoRecoveryMain(confByIndex(2)); + + /* + * start main1, make sure all the components are started and main1 is + * the current Auditor + */ + PulsarMetadataClientDriver pulsarMetadataClientDriver1 = startAutoRecoveryMain(main1); + ZooKeeper zk1 = getZk(pulsarMetadataClientDriver1); + + // Wait until auditor gets elected + for (int i = 0; i < 10; i++) { + try { + if (main1.auditorElector.getCurrentAuditor() != null) { + break; + } else { + Thread.sleep(1000); + } + } catch (IOException e) { + Thread.sleep(1000); + } + } + BookieId currentAuditor = main1.auditorElector.getCurrentAuditor(); + assertNotNull(currentAuditor); + Auditor auditor1 = main1.auditorElector.getAuditor(); + assertEquals("Current Auditor should be AR1", currentAuditor, BookieImpl.getBookieId(confByIndex(0))); + Awaitility.waitAtMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertNotNull(auditor1); + assertTrue("Auditor of AR1 should be running", auditor1.isRunning()); + }); + + + /* + * start main2 and main3 + */ + PulsarMetadataClientDriver pulsarMetadataClientDriver2 = startAutoRecoveryMain(main2); + ZooKeeper zk2 = getZk(pulsarMetadataClientDriver2); + + PulsarMetadataClientDriver pulsarMetadataClientDriver3 = startAutoRecoveryMain(main3); + ZooKeeper zk3 = getZk(pulsarMetadataClientDriver3); + + + /* + * make sure AR1 is still the current Auditor and AR2's and AR3's + * auditors are not running. + */ + assertEquals("Current Auditor should still be AR1", currentAuditor, BookieImpl.getBookieId(confByIndex(0))); + Awaitility.await().untilAsserted(() -> { + assertTrue("AR2's Auditor should not be running", (main2.auditorElector.getAuditor() == null + || !main2.auditorElector.getAuditor().isRunning())); + assertTrue("AR3's Auditor should not be running", (main3.auditorElector.getAuditor() == null + || !main3.auditorElector.getAuditor().isRunning())); + }); + + + /* + * expire zk2 and zk1 sessions. + */ + zkUtil.expireSession(zk2); + zkUtil.expireSession(zk1); + + /* + * wait for some time for all the components of AR1 and AR2 are + * shutdown. + */ + for (int i = 0; i < 10; i++) { + if (!main1.auditorElector.isRunning() && !main1.replicationWorker.isRunning() + && !main1.isAutoRecoveryRunning() && !main2.auditorElector.isRunning() + && !main2.replicationWorker.isRunning() && !main2.isAutoRecoveryRunning()) { + break; + } + Thread.sleep(1000); + } + + /* + * the AR3 should be current auditor. + */ + currentAuditor = main3.auditorElector.getCurrentAuditor(); + assertEquals("Current Auditor should be AR3", currentAuditor, BookieImpl.getBookieId(confByIndex(2))); + Awaitility.await().untilAsserted(() -> { + assertNotNull(main3.auditorElector.getAuditor()); + assertTrue("Auditor of AR3 should be running", main3.auditorElector.getAuditor().isRunning()); + }); + + Awaitility.waitAtMost(100, TimeUnit.SECONDS).untilAsserted(() -> { + /* + * since AR3 is current auditor, AR1's auditor should not be running + * anymore. + */ + assertFalse("AR1's auditor should not be running", auditor1.isRunning()); + + /* + * components of AR2 and AR3 should not be running since zk1 and zk2 + * sessions are expired. + */ + assertFalse("Elector1 should have shutdown", main1.auditorElector.isRunning()); + assertFalse("RW1 should have shutdown", main1.replicationWorker.isRunning()); + assertFalse("AR1 should have shutdown", main1.isAutoRecoveryRunning()); + assertFalse("Elector2 should have shutdown", main2.auditorElector.isRunning()); + assertFalse("RW2 should have shutdown", main2.replicationWorker.isRunning()); + assertFalse("AR2 should have shutdown", main2.isAutoRecoveryRunning()); + }); + + } + + /* + * start autoRecoveryMain and make sure all its components are running and + * myVote node is existing + */ + PulsarMetadataClientDriver startAutoRecoveryMain(AutoRecoveryMain autoRecoveryMain) throws Exception { + autoRecoveryMain.start(); + PulsarMetadataClientDriver pulsarMetadataClientDriver = (PulsarMetadataClientDriver) autoRecoveryMain.bkc + .getMetadataClientDriver(); + TestUtils.assertEventuallyTrue("autoRecoveryMain components should be running", + () -> autoRecoveryMain.auditorElector.isRunning() + && autoRecoveryMain.replicationWorker.isRunning() && autoRecoveryMain.isAutoRecoveryRunning()); + return pulsarMetadataClientDriver; + } + + private ZooKeeper getZk(PulsarMetadataClientDriver pulsarMetadataClientDriver) throws Exception { + PulsarLedgerManagerFactory pulsarLedgerManagerFactory = + (PulsarLedgerManagerFactory) pulsarMetadataClientDriver.getLedgerManagerFactory(); + Field field = pulsarLedgerManagerFactory.getClass().getDeclaredField("store"); + field.setAccessible(true); + ZKMetadataStore zkMetadataStore = (ZKMetadataStore) field.get(pulsarLedgerManagerFactory); + return zkMetadataStore.getZkClient(); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java new file mode 100644 index 0000000000000..c8c76302b89e1 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java @@ -0,0 +1,652 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertTrue; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.SortedMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.BookKeeperTestClient; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataClientDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.Watcher.Event.EventType; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Integration tests verifies the complete functionality of the + * Auditor-rereplication process: Auditor will publish the bookie failures, + * consequently ReplicationWorker will get the notifications and act on it. + */ +public class BookieAutoRecoveryTest extends BookKeeperClusterTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(BookieAutoRecoveryTest.class); + private static final byte[] PASSWD = "admin".getBytes(); + private static final byte[] data = "TESTDATA".getBytes(); + private static final String openLedgerRereplicationGracePeriod = "3000"; // milliseconds + + private DigestType digestType; + private MetadataClientDriver metadataClientDriver; + private LedgerManagerFactory mFactory; + private LedgerUnderreplicationManager underReplicationManager; + private LedgerManager ledgerManager; + private OrderedScheduler scheduler; + + private final String underreplicatedPath = "/ledgers/underreplication/ledgers"; + + public BookieAutoRecoveryTest() throws Exception { + super(3); + + baseConf.setLedgerManagerFactoryClassName( + "org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + baseConf.setOpenLedgerRereplicationGracePeriod(openLedgerRereplicationGracePeriod); + baseConf.setRwRereplicateBackoffMs(500); + baseClientConf.setLedgerManagerFactoryClassName( + "org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + this.digestType = DigestType.MAC; + setAutoRecoveryEnabled(true); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + baseClientConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + + scheduler = OrderedScheduler.newSchedulerBuilder() + .name("test-scheduler") + .numThreads(1) + .build(); + + metadataClientDriver = MetadataDrivers.getClientDriver( + URI.create(baseClientConf.getMetadataServiceUri())); + metadataClientDriver.initialize( + baseClientConf, + scheduler, + NullStatsLogger.INSTANCE, + Optional.empty()); + + // initialize urReplicationManager + mFactory = metadataClientDriver.getLedgerManagerFactory(); + underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + ledgerManager = mFactory.newLedgerManager(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + + if (null != underReplicationManager) { + underReplicationManager.close(); + underReplicationManager = null; + } + if (null != ledgerManager) { + ledgerManager.close(); + ledgerManager = null; + } + if (null != metadataClientDriver) { + metadataClientDriver.close(); + metadataClientDriver = null; + } + if (null != scheduler) { + scheduler.shutdown(); + } + } + + /** + * Test verifies publish urLedger by Auditor and replication worker is + * picking up the entries and finishing the rereplication of open ledger. + */ + @Test + public void testOpenLedgers() throws Exception { + List listOfLedgerHandle = createLedgersAndAddEntries(1, 5); + LedgerHandle lh = listOfLedgerHandle.get(0); + int ledgerReplicaIndex = 0; + BookieId replicaToKillAddr = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + final String urLedgerZNode = getUrLedgerZNode(lh); + ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr); + + CountDownLatch latch = new CountDownLatch(1); + assertNull("UrLedger already exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + latch = new CountDownLatch(1); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + // starting the replication service, so that he will be able to act as + // target bookie + startNewBookie(); + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(0), ledgerReplicaIndex); + } + + /** + * Test verifies publish urLedger by Auditor and replication worker is + * picking up the entries and finishing the rereplication of closed ledgers. + */ + @Test + public void testClosedLedgers() throws Exception { + List listOfReplicaIndex = new ArrayList(); + List listOfLedgerHandle = createLedgersAndAddEntries(1, 5); + closeLedgers(listOfLedgerHandle); + LedgerHandle lhandle = listOfLedgerHandle.get(0); + int ledgerReplicaIndex = 0; + BookieId replicaToKillAddr = lhandle.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr); + listOfReplicaIndex.add(ledgerReplicaIndex); + assertNull("UrLedger already exists!", + watchUrLedgerNode(getUrLedgerZNode(lh), latch)); + } + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + + // Again watching the urLedger znode to know the replication status + latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + String urLedgerZNode = getUrLedgerZNode(lh); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + } + + // starting the replication service, so that he will be able to act as + // target bookie + startNewBookie(); + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + + // waiting to finish replication + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + for (int index = 0; index < listOfLedgerHandle.size(); index++) { + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(index), + listOfReplicaIndex.get(index)); + } + } + + /** + * Test stopping replica service while replication in progress. Considering + * when there is an exception will shutdown Auditor and RW processes. After + * restarting should be able to finish the re-replication activities + */ + @Test + public void testStopWhileReplicationInProgress() throws Exception { + int numberOfLedgers = 2; + List listOfReplicaIndex = new ArrayList(); + List listOfLedgerHandle = createLedgersAndAddEntries( + numberOfLedgers, 5); + closeLedgers(listOfLedgerHandle); + LedgerHandle handle = listOfLedgerHandle.get(0); + BookieId replicaToKillAddr = handle.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + LOG.info("Killing Bookie:" + replicaToKillAddr); + + // Each ledger, there will be two events : create urLedger and after + // rereplication delete urLedger + CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size()); + for (int i = 0; i < listOfLedgerHandle.size(); i++) { + final String urLedgerZNode = getUrLedgerZNode(listOfLedgerHandle + .get(i)); + assertNull("UrLedger already exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + int replicaIndexInLedger = getReplicaIndexInLedger( + listOfLedgerHandle.get(i), replicaToKillAddr); + listOfReplicaIndex.add(replicaIndexInLedger); + } + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + + // Again watching the urLedger znode to know the replication status + latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + String urLedgerZNode = getUrLedgerZNode(lh); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + } + + // starting the replication service, so that he will be able to act as + // target bookie + startNewBookie(); + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + while (true) { + if (latch.getCount() < numberOfLedgers || latch.getCount() <= 0) { + stopReplicationService(); + LOG.info("Latch Count is:" + latch.getCount()); + break; + } + // grace period to take breath + Thread.sleep(1000); + } + + startReplicationService(); + + LOG.info("Waiting to finish rereplication processes"); + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + for (int index = 0; index < listOfLedgerHandle.size(); index++) { + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(index), + listOfReplicaIndex.get(index)); + } + } + + /** + * Verify the published urledgers of deleted ledgers(those ledgers where + * deleted after publishing as urledgers by Auditor) should be cleared off + * by the newly selected replica bookie. + */ + @Test + public void testNoSuchLedgerExists() throws Exception { + List listOfLedgerHandle = createLedgersAndAddEntries(2, 5); + CountDownLatch latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + assertNull("UrLedger already exists!", + watchUrLedgerNode(getUrLedgerZNode(lh), latch)); + } + BookieId replicaToKillAddr = listOfLedgerHandle.get(0) + .getLedgerMetadata().getAllEnsembles() + .get(0L).get(0); + killBookie(replicaToKillAddr); + replicaToKillAddr = listOfLedgerHandle.get(0) + .getLedgerMetadata().getAllEnsembles() + .get(0L).get(0); + killBookie(replicaToKillAddr); + // waiting to publish urLedger znode by Auditor + latch.await(); + + latch = new CountDownLatch(listOfLedgerHandle.size()); + for (LedgerHandle lh : listOfLedgerHandle) { + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(getUrLedgerZNode(lh), latch)); + } + + // delete ledgers + for (LedgerHandle lh : listOfLedgerHandle) { + bkc.deleteLedger(lh.getId()); + } + startNewBookie(); + + // waiting to delete published urledgers, since it doesn't exists + latch.await(); + + for (LedgerHandle lh : listOfLedgerHandle) { + assertNull("UrLedger still exists after rereplication", + watchUrLedgerNode(getUrLedgerZNode(lh), latch)); + } + } + + /** + * Test that if a empty ledger loses the bookie not in the quorum for entry 0, it will + * still be openable when it loses enough bookies to lose a whole quorum. + */ + @Test + public void testEmptyLedgerLosesQuorumEventually() throws Exception { + LedgerHandle lh = bkc.createLedger(3, 2, 2, DigestType.CRC32, PASSWD); + CountDownLatch latch = new CountDownLatch(1); + String urZNode = getUrLedgerZNode(lh); + watchUrLedgerNode(urZNode, latch); + + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(2); + LOG.info("Killing last bookie, {}, in ensemble {}", replicaToKill, + lh.getLedgerMetadata().getAllEnsembles().get(0L)); + killBookie(replicaToKill); + startNewBookie(); + + getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs + + assertTrue("Should be marked as underreplicated", latch.await(5, TimeUnit.SECONDS)); + latch = new CountDownLatch(1); + Stat s = watchUrLedgerNode(urZNode, latch); // should be marked as replicated + if (s != null) { + assertTrue("Should be marked as replicated", latch.await(15, TimeUnit.SECONDS)); + } + + replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(1); + LOG.info("Killing second bookie, {}, in ensemble {}", replicaToKill, + lh.getLedgerMetadata().getAllEnsembles().get(0L)); + killBookie(replicaToKill); + + getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs + + assertTrue("Should be marked as underreplicated", latch.await(5, TimeUnit.SECONDS)); + latch = new CountDownLatch(1); + s = watchUrLedgerNode(urZNode, latch); // should be marked as replicated + + startNewBookie(); + getAuditor(10, TimeUnit.SECONDS).submitAuditTask().get(); // ensure auditor runs + + if (s != null) { + assertTrue("Should be marked as replicated", latch.await(20, TimeUnit.SECONDS)); + } + + // should be able to open ledger without issue + bkc.openLedger(lh.getId(), DigestType.CRC32, PASSWD); + } + + /** + * Test verifies bookie recovery, the host (recorded via ipaddress in + * ledgermetadata). + */ + @Test + public void testLedgerMetadataContainsIpAddressAsBookieID() + throws Exception { + stopBKCluster(); + bkc = new BookKeeperTestClient(baseClientConf); + // start bookie with useHostNameAsBookieID=false, as old bookie + ServerConfiguration serverConf1 = newServerConfiguration(); + // start 2 more bookies with useHostNameAsBookieID=true + ServerConfiguration serverConf2 = newServerConfiguration(); + serverConf2.setUseHostNameAsBookieID(true); + ServerConfiguration serverConf3 = newServerConfiguration(); + serverConf3.setUseHostNameAsBookieID(true); + startAndAddBookie(serverConf1); + startAndAddBookie(serverConf2); + startAndAddBookie(serverConf3); + + List listOfLedgerHandle = createLedgersAndAddEntries(1, 5); + LedgerHandle lh = listOfLedgerHandle.get(0); + int ledgerReplicaIndex = 0; + final SortedMap> ensembles = lh.getLedgerMetadata().getAllEnsembles(); + final List bkAddresses = ensembles.get(0L); + BookieId replicaToKillAddr = bkAddresses.get(0); + for (BookieId bookieSocketAddress : bkAddresses) { + if (!isCreatedFromIp(bookieSocketAddress)) { + replicaToKillAddr = bookieSocketAddress; + LOG.info("Kill bookie which has registered using hostname"); + break; + } + } + + final String urLedgerZNode = getUrLedgerZNode(lh); + ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr); + + CountDownLatch latch = new CountDownLatch(1); + assertNull("UrLedger already exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + latch = new CountDownLatch(1); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + // starting the replication service, so that he will be able to act as + // target bookie + ServerConfiguration serverConf = newServerConfiguration(); + serverConf.setUseHostNameAsBookieID(false); + startAndAddBookie(serverConf); + + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(0), ledgerReplicaIndex); + + } + + /** + * Test verifies bookie recovery, the host (recorded via useHostName in + * ledgermetadata). + */ + @Test + public void testLedgerMetadataContainsHostNameAsBookieID() + throws Exception { + stopBKCluster(); + + bkc = new BookKeeperTestClient(baseClientConf); + // start bookie with useHostNameAsBookieID=false, as old bookie + ServerConfiguration serverConf1 = newServerConfiguration(); + // start 2 more bookies with useHostNameAsBookieID=true + ServerConfiguration serverConf2 = newServerConfiguration(); + serverConf2.setUseHostNameAsBookieID(true); + ServerConfiguration serverConf3 = newServerConfiguration(); + serverConf3.setUseHostNameAsBookieID(true); + startAndAddBookie(serverConf1); + startAndAddBookie(serverConf2); + startAndAddBookie(serverConf3); + + List listOfLedgerHandle = createLedgersAndAddEntries(1, 5); + LedgerHandle lh = listOfLedgerHandle.get(0); + int ledgerReplicaIndex = 0; + final SortedMap> ensembles = lh.getLedgerMetadata().getAllEnsembles(); + final List bkAddresses = ensembles.get(0L); + BookieId replicaToKillAddr = bkAddresses.get(0); + for (BookieId bookieSocketAddress : bkAddresses) { + if (isCreatedFromIp(bookieSocketAddress)) { + replicaToKillAddr = bookieSocketAddress; + LOG.info("Kill bookie which has registered using ipaddress"); + break; + } + } + + final String urLedgerZNode = getUrLedgerZNode(lh); + ledgerReplicaIndex = getReplicaIndexInLedger(lh, replicaToKillAddr); + + CountDownLatch latch = new CountDownLatch(1); + assertNull("UrLedger already exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + LOG.info("Killing Bookie :" + replicaToKillAddr); + killBookie(replicaToKillAddr); + + // waiting to publish urLedger znode by Auditor + latch.await(); + latch = new CountDownLatch(1); + LOG.info("Watching on urLedgerPath:" + urLedgerZNode + + " to know the status of rereplication process"); + assertNotNull("UrLedger doesn't exists!", + watchUrLedgerNode(urLedgerZNode, latch)); + + // creates new bkclient + bkc = new BookKeeperTestClient(baseClientConf); + // starting the replication service, so that he will be able to act as + // target bookie + ServerConfiguration serverConf = newServerConfiguration(); + serverConf.setUseHostNameAsBookieID(true); + startAndAddBookie(serverConf); + + int newBookieIndex = lastBookieIndex(); + BookieServer newBookieServer = serverByIndex(newBookieIndex); + + if (LOG.isDebugEnabled()) { + LOG.debug("Waiting to finish the replication of failed bookie : " + + replicaToKillAddr); + } + latch.await(); + + // grace period to update the urledger metadata in zookeeper + LOG.info("Waiting to update the urledger metadata in zookeeper"); + + verifyLedgerEnsembleMetadataAfterReplication(newBookieServer, + listOfLedgerHandle.get(0), ledgerReplicaIndex); + + } + + private int getReplicaIndexInLedger(LedgerHandle lh, BookieId replicaToKill) { + SortedMap> ensembles = lh.getLedgerMetadata().getAllEnsembles(); + int ledgerReplicaIndex = -1; + for (BookieId addr : ensembles.get(0L)) { + ++ledgerReplicaIndex; + if (addr.equals(replicaToKill)) { + break; + } + } + return ledgerReplicaIndex; + } + + private void verifyLedgerEnsembleMetadataAfterReplication( + BookieServer newBookieServer, LedgerHandle lh, + int ledgerReplicaIndex) throws Exception { + LedgerHandle openLedger = bkc + .openLedger(lh.getId(), digestType, PASSWD); + + BookieId inetSocketAddress = openLedger.getLedgerMetadata().getAllEnsembles().get(0L) + .get(ledgerReplicaIndex); + assertEquals("Rereplication has been failed and ledgerReplicaIndex :" + + ledgerReplicaIndex, newBookieServer.getBookieId(), + inetSocketAddress); + openLedger.close(); + } + + private void closeLedgers(List listOfLedgerHandle) + throws InterruptedException, BKException { + for (LedgerHandle lh : listOfLedgerHandle) { + lh.close(); + } + } + + private List createLedgersAndAddEntries(int numberOfLedgers, + int numberOfEntries) + throws InterruptedException, BKException { + List listOfLedgerHandle = new ArrayList( + numberOfLedgers); + for (int index = 0; index < numberOfLedgers; index++) { + LedgerHandle lh = bkc.createLedger(3, 3, digestType, PASSWD); + listOfLedgerHandle.add(lh); + for (int i = 0; i < numberOfEntries; i++) { + lh.addEntry(data); + } + } + return listOfLedgerHandle; + } + + private String getUrLedgerZNode(LedgerHandle lh) { + return ZkLedgerUnderreplicationManager.getUrLedgerZnode( + underreplicatedPath, lh.getId()); + } + + private Stat watchUrLedgerNode(final String znode, + final CountDownLatch latch) throws KeeperException, + InterruptedException { + return zkc.exists(znode, new Watcher() { + @Override + public void process(WatchedEvent event) { + if (event.getType() == EventType.NodeDeleted) { + LOG.info("Received Ledger rereplication completion event :" + + event.getType()); + latch.countDown(); + } + if (event.getType() == EventType.NodeCreated) { + LOG.info("Received urLedger publishing event :" + + event.getType()); + latch.countDown(); + } + } + }); + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java new file mode 100644 index 0000000000000..eb9f95ffdf7a5 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper.DigestType; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.meta.LayoutManager; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests verifies bookie vs ledger mapping generating by the BookieLedgerIndexer. + */ +public class BookieLedgerIndexTest extends BookKeeperClusterTestCase { + + // Depending on the taste, select the amount of logging + // by decommenting one of the two lines below + // private final static Logger LOG = Logger.getRootLogger(); + private static final Logger LOG = LoggerFactory + .getLogger(BookieLedgerIndexTest.class); + + private Random rng; // Random Number Generator + private ArrayList entries; // generated entries + private final DigestType digestType = DigestType.CRC32; + private int numberOfLedgers = 3; + private List ledgerList; + private LedgerManagerFactory newLedgerManagerFactory; + private LedgerManager ledgerManager; + + public BookieLedgerIndexTest() throws Exception { + this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + } + + BookieLedgerIndexTest(String ledgerManagerFactory) throws Exception { + super(3); + LOG.info("Running test case using ledger manager : " + + ledgerManagerFactory); + // set ledger manager name + baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory); + baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + public void setUp() throws Exception { + super.setUp(); + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + rng = new Random(System.currentTimeMillis()); // Initialize the Random + // Number Generator + entries = new ArrayList(); // initialize the entries list + ledgerList = new ArrayList(3); + + String ledgersRoot = "/ledgers"; + String storeUri = metadataServiceUri.replaceAll("zk://", "").replaceAll("/ledgers", ""); + MetadataStoreExtended store = MetadataStoreExtended.create(storeUri, + MetadataStoreConfig.builder().fsyncEnable(false).build()); + LayoutManager layoutManager = new PulsarLayoutManager(store, ledgersRoot); + newLedgerManagerFactory = new PulsarLedgerManagerFactory(); + + ClientConfiguration conf = new ClientConfiguration(); + conf.setZkLedgersRootPath(ledgersRoot); + newLedgerManagerFactory.initialize(conf, layoutManager, 1); + ledgerManager = newLedgerManagerFactory.newLedgerManager(); + } + + @AfterMethod + public void tearDown() throws Exception { + super.tearDown(); + if (null != newLedgerManagerFactory) { + newLedgerManagerFactory.close(); + newLedgerManagerFactory = null; + } + if (null != ledgerManager) { + ledgerManager.close(); + ledgerManager = null; + } + } + + /** + * Verify the bookie-ledger mapping with minimum number of bookies and few + * ledgers. + */ + @Test + public void testSimpleBookieLedgerMapping() throws Exception { + + for (int i = 0; i < numberOfLedgers; i++) { + createAndAddEntriesToLedger().close(); + } + + BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer( + ledgerManager); + + Map> bookieToLedgerIndex = bookieLedgerIndex + .getBookieToLedgerIndex(); + + assertEquals("Missed few bookies in the bookie-ledger mapping!", 3, + bookieToLedgerIndex.size()); + Collection> bk2ledgerEntry = bookieToLedgerIndex.values(); + for (Set ledgers : bk2ledgerEntry) { + assertEquals("Missed few ledgers in the bookie-ledger mapping!", 3, + ledgers.size()); + for (Long ledgerId : ledgers) { + assertTrue("Unknown ledger-bookie mapping", ledgerList + .contains(ledgerId)); + } + } + } + + /** + * Verify ledger index with failed bookies and throws exception. + */ + @SuppressWarnings("deprecation") +// @Test +// public void testWithoutZookeeper() throws Exception { +// // This test case is for ledger metadata that stored in ZooKeeper. As +// // far as MSLedgerManagerFactory, ledger metadata are stored in other +// // storage. So this test is not suitable for MSLedgerManagerFactory. +// if (newLedgerManagerFactory instanceof org.apache.bookkeeper.meta.MSLedgerManagerFactory) { +// return; +// } +// +// for (int i = 0; i < numberOfLedgers; i++) { +// createAndAddEntriesToLedger().close(); +// } +// +// BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer( +// ledgerManager); +// stopZKCluster(); +// try { +// bookieLedgerIndex.getBookieToLedgerIndex(); +// fail("Must throw exception as zookeeper are not running!"); +// } catch (BKAuditException bkAuditException) { +// // expected behaviour +// } +// } + + /** + * Verify indexing with multiple ensemble reformation. + */ + @Test + public void testEnsembleReformation() throws Exception { + try { + LedgerHandle lh1 = createAndAddEntriesToLedger(); + LedgerHandle lh2 = createAndAddEntriesToLedger(); + + startNewBookie(); + shutdownBookie(lastBookieIndex() - 1); + + // add few more entries after ensemble reformation + for (int i = 0; i < 10; i++) { + ByteBuffer entry = ByteBuffer.allocate(4); + entry.putInt(rng.nextInt(Integer.MAX_VALUE)); + entry.position(0); + + entries.add(entry.array()); + lh1.addEntry(entry.array()); + lh2.addEntry(entry.array()); + } + + BookieLedgerIndexer bookieLedgerIndex = new BookieLedgerIndexer( + ledgerManager); + + Map> bookieToLedgerIndex = bookieLedgerIndex + .getBookieToLedgerIndex(); + assertEquals("Missed few bookies in the bookie-ledger mapping!", 4, + bookieToLedgerIndex.size()); + Collection> bk2ledgerEntry = bookieToLedgerIndex.values(); + for (Set ledgers : bk2ledgerEntry) { + assertEquals( + "Missed few ledgers in the bookie-ledger mapping!", 2, + ledgers.size()); + for (Long ledgerNode : ledgers) { + assertTrue("Unknown ledger-bookie mapping", ledgerList + .contains(ledgerNode)); + } + } + } catch (BKException e) { + LOG.error("Test failed", e); + fail("Test failed due to BookKeeper exception"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.error("Test failed", e); + fail("Test failed due to interruption"); + } + } + + private void shutdownBookie(int bkShutdownIndex) throws Exception { + killBookie(bkShutdownIndex); + } + + private LedgerHandle createAndAddEntriesToLedger() throws BKException, + InterruptedException { + int numEntriesToWrite = 20; + // Create a ledger + LedgerHandle lh = bkc.createLedger(digestType, "admin".getBytes()); + LOG.info("Ledger ID: " + lh.getId()); + for (int i = 0; i < numEntriesToWrite; i++) { + ByteBuffer entry = ByteBuffer.allocate(4); + entry.putInt(rng.nextInt(Integer.MAX_VALUE)); + entry.position(0); + + entries.add(entry.array()); + lh.addEntry(entry.array()); + } + ledgerList.add(lh.getId()); + return lh; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java new file mode 100644 index 0000000000000..4360bf3254675 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ReplicationTestUtil.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import java.util.List; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; + +/** + * Utility class for replication tests. + */ +public class ReplicationTestUtil { + + /** + * Checks whether ledger is in under-replication. + */ + public static boolean isLedgerInUnderReplication(ZooKeeper zkc, long id, + String basePath) throws KeeperException, InterruptedException { + List children; + try { + children = zkc.getChildren(basePath, true); + } catch (KeeperException.NoNodeException nne) { + return false; + } + + boolean isMatched = false; + for (String child : children) { + if (child.startsWith("urL") && child.contains(String.valueOf(id))) { + isMatched = true; + break; + } else { + String path = basePath + '/' + child; + try { + if (zkc.getChildren(path, false).size() > 0) { + isMatched = isLedgerInUnderReplication(zkc, id, path); + } + } catch (KeeperException.NoNodeException nne) { + return false; + } + } + + } + return isMatched; + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java new file mode 100644 index 0000000000000..8a2e7f2747a22 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; +import java.util.Enumeration; +import java.util.List; +import java.util.Map.Entry; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.util.BookKeeperConstants; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test auto recovery. + */ +public class TestAutoRecoveryAlongWithBookieServers extends + BookKeeperClusterTestCase { + + private String basePath = ""; + + public TestAutoRecoveryAlongWithBookieServers() throws Exception { + super(3); + setAutoRecoveryEnabled(true); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + basePath = BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH + '/' + + BookKeeperConstants.UNDER_REPLICATION_NODE + + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH; + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + @Override + protected void startBKCluster(String metadataServiceUri) throws Exception { + super.startBKCluster(metadataServiceUri.replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + } + + /** + * Tests that the auto recovery service along with Bookie servers itself. + */ + @Test + public void testAutoRecoveryAlongWithBookieServers() throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + "testpasswd".getBytes()); + byte[] testData = "testBuiltAutoRecovery".getBytes(); + + for (int i = 0; i < 10; i++) { + lh.addEntry(testData); + } + lh.close(); + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), + basePath)) { + Thread.sleep(100); + } + + // Killing all bookies except newly replicated bookie + for (Entry> entry : + lh.getLedgerMetadata().getAllEnsembles().entrySet()) { + List bookies = entry.getValue(); + for (BookieId bookie : bookies) { + if (bookie.equals(newBkAddr)) { + continue; + } + killBookie(bookie); + } + } + + // Should be able to read the entries from 0-9 + LedgerHandle lhs = bkc.openLedgerNoRecovery(lh.getId(), + BookKeeper.DigestType.CRC32, "testpasswd".getBytes()); + Enumeration entries = lhs.readEntries(0, 9); + assertTrue("Should have the elements", entries.hasMoreElements()); + while (entries.hasMoreElements()) { + LedgerEntry entry = entries.nextElement(); + assertEquals("testBuiltAutoRecovery", new String(entry.getEntry())); + } + } +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java new file mode 100644 index 0000000000000..ca02f91d1de36 --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -0,0 +1,1249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.replication.ReplicationStats.AUDITOR_SCOPE; +import static org.apache.bookkeeper.replication.ReplicationStats.NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION; +import static org.apache.bookkeeper.replication.ReplicationStats.REPLICATION_SCOPE; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertFalse; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertNull; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; +import io.netty.util.HashedWheelTimer; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URI; +import java.net.UnknownHostException; +import java.util.Enumeration; +import java.util.List; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.TimerTask; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import lombok.Cleanup; +import org.apache.bookkeeper.bookie.BookieImpl; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.client.BookKeeperTestClient; +import org.apache.bookkeeper.client.ClientUtil; +import org.apache.bookkeeper.client.EnsemblePlacementPolicy; +import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.ZoneawareEnsemblePlacementPolicy; +import org.apache.bookkeeper.client.api.LedgerMetadata; +import org.apache.bookkeeper.common.util.OrderedScheduler; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.feature.FeatureProvider; +import org.apache.bookkeeper.meta.LedgerManager; +import org.apache.bookkeeper.meta.LedgerManagerFactory; +import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.MetadataBookieDriver; +import org.apache.bookkeeper.meta.MetadataClientDriver; +import org.apache.bookkeeper.meta.MetadataDrivers; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.exceptions.MetadataException; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.DNSToSwitchMapping; +import org.apache.bookkeeper.proto.BookieAddressResolver; +import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException; +import org.apache.bookkeeper.stats.Counter; +import org.apache.bookkeeper.stats.Gauge; +import org.apache.bookkeeper.stats.NullStatsLogger; +import org.apache.bookkeeper.stats.StatsLogger; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; +import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; +import org.apache.bookkeeper.util.BookKeeperConstants; +import org.apache.bookkeeper.util.StaticDNSResolver; +import org.apache.bookkeeper.zookeeper.ZooKeeperClient; +import org.apache.commons.lang3.mutable.MutableObject; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; +import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.data.Stat; +import org.awaitility.Awaitility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Test the ReplicationWroker, where it has to replicate the fragments from + * failed Bookies to given target Bookie. + */ +public class TestReplicationWorker extends BookKeeperClusterTestCase { + + private static final byte[] TESTPASSWD = "testpasswd".getBytes(); + private static final Logger LOG = LoggerFactory + .getLogger(TestReplicationWorker.class); + private String basePath = ""; + private String baseLockPath = ""; + private MetadataBookieDriver driver; + private LedgerManagerFactory mFactory; + private LedgerUnderreplicationManager underReplicationManager; + private LedgerManager ledgerManager; + private static byte[] data = "TestReplicationWorker".getBytes(); + private OrderedScheduler scheduler; + private String zkLedgersRootPath; + + public TestReplicationWorker() throws Exception { + this("org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory"); + } + + TestReplicationWorker(String ledgerManagerFactory) throws Exception { + super(3, 300); + LOG.info("Running test case using ledger manager : " + + ledgerManagerFactory); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver"); + Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); + // set ledger manager name + baseConf.setLedgerManagerFactoryClassName(ledgerManagerFactory); + baseClientConf.setLedgerManagerFactoryClassName(ledgerManagerFactory); + baseConf.setRereplicationEntryBatchSize(3); + baseConf.setZkTimeout(7000); + baseConf.setZkRetryBackoffMaxMs(500); + baseConf.setZkRetryBackoffStartMs(10); + } + + @BeforeMethod + @Override + public void setUp() throws Exception { + super.setUp(); + zkLedgersRootPath = ZKMetadataDriverBase.resolveZkLedgersRootPath(baseClientConf); + basePath = zkLedgersRootPath + '/' + + BookKeeperConstants.UNDER_REPLICATION_NODE + + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH; + baseLockPath = zkLedgersRootPath + '/' + + BookKeeperConstants.UNDER_REPLICATION_NODE + + "/locks"; + baseClientConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + baseConf.setMetadataServiceUri( + zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); + this.scheduler = OrderedScheduler.newSchedulerBuilder() + .name("test-scheduler") + .numThreads(1) + .build(); + + this.driver = MetadataDrivers.getBookieDriver( + URI.create(baseConf.getMetadataServiceUri())); + this.driver.initialize( + baseConf, + NullStatsLogger.INSTANCE); + // initialize urReplicationManager + mFactory = driver.getLedgerManagerFactory(); + ledgerManager = mFactory.newLedgerManager(); + underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + } + + @AfterMethod + @Override + public void tearDown() throws Exception { + super.tearDown(); + if (null != ledgerManager) { + ledgerManager.close(); + ledgerManager = null; + } + if (null != underReplicationManager) { + underReplicationManager.close(); + underReplicationManager = null; + } + if (null != driver) { + driver.close(); + } + if (null != scheduler) { + scheduler.shutdown(); + scheduler = null; + } + if (null != mFactory) { + mFactory.close(); + } + } + + /** + * Tests that replication worker should replicate the failed bookie + * fragments to target bookie given to the worker. + */ + @Test + public void testRWShouldReplicateFragmentsToTargetBookie() throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKill); + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + + ReplicationWorker rw = new ReplicationWorker(baseConf); + + rw.start(); + try { + + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + + killAllBookies(lh, newBkAddr); + + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + } finally { + rw.shutdown(); + } + } + + /** + * Tests that replication worker should retry for replication until enough + * bookies available for replication. + */ + @Test + public void testRWShouldRetryUntilThereAreEnoughBksAvailableForReplication() + throws Exception { + LedgerHandle lh = bkc.createLedger(1, 1, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + lh.close(); + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + LOG.info("Killing Bookie : {}", replicaToKill); + ServerConfiguration killedBookieConfig = killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr :" + newBkAddr); + + killAllBookies(lh, newBkAddr); + ReplicationWorker rw = new ReplicationWorker(baseConf); + + rw.start(); + try { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + int counter = 30; + while (counter-- > 0) { + assertTrue("Expecting that replication should not complete", + ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)); + Thread.sleep(100); + } + // restart killed bookie + startAndAddBookie(killedBookieConfig); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + } finally { + rw.shutdown(); + } + } + + /** + * Tests that replication worker1 should take one fragment replication and + * other replication worker also should compete for the replication. + */ + @Test + public void test2RWsShouldCompeteForReplicationOf2FragmentsAndCompleteReplication() + throws Exception { + LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + lh.close(); + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + LOG.info("Killing Bookie : {}", replicaToKill); + ServerConfiguration killedBookieConfig = killBookie(replicaToKill); + + killAllBookies(lh, null); + // Starte RW1 + BookieId newBkAddr1 = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr1); + ReplicationWorker rw1 = new ReplicationWorker(baseConf); + + // Starte RW2 + BookieId newBkAddr2 = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr2); + ReplicationWorker rw2 = new ReplicationWorker(baseConf); + rw1.start(); + rw2.start(); + + try { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + int counter = 10; + while (counter-- > 0) { + assertTrue("Expecting that replication should not complete", + ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)); + Thread.sleep(100); + } + // restart killed bookie + startAndAddBookie(killedBookieConfig); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + } finally { + rw1.shutdown(); + rw2.shutdown(); + } + } + + /** + * Tests that Replication worker should clean the leadger under replication + * node of the ledger already deleted. + */ + @Test + public void testRWShouldCleanTheLedgerFromUnderReplicationIfLedgerAlreadyDeleted() + throws Exception { + LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + lh.close(); + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + LOG.info("Killing Bookie : {}", replicaToKill); + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + ReplicationWorker rw = new ReplicationWorker(baseConf); + rw.start(); + + try { + bkc.deleteLedger(lh.getId()); // Deleting the ledger + // Also mark ledger as in UnderReplication + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + } finally { + rw.shutdown(); + } + + } + + @Test + public void testMultipleLedgerReplicationWithReplicationWorker() + throws Exception { + // Ledger1 + LedgerHandle lh1 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh1.addEntry(data); + } + BookieId replicaToKillFromFirstLedger = lh1.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKillFromFirstLedger); + + // Ledger2 + LedgerHandle lh2 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh2.addEntry(data); + } + BookieId replicaToKillFromSecondLedger = lh2.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKillFromSecondLedger); + + // Kill ledger1 + killBookie(replicaToKillFromFirstLedger); + lh1.close(); + // Kill ledger2 + killBookie(replicaToKillFromFirstLedger); + lh2.close(); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + + ReplicationWorker rw = new ReplicationWorker(baseConf); + + rw.start(); + try { + + // Mark ledger1 and 2 as underreplicated + underReplicationManager.markLedgerUnderreplicated(lh1.getId(), + replicaToKillFromFirstLedger.toString()); + underReplicationManager.markLedgerUnderreplicated(lh2.getId(), + replicaToKillFromSecondLedger.toString()); + + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh1 + .getId(), basePath)) { + Thread.sleep(100); + } + + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh2 + .getId(), basePath)) { + Thread.sleep(100); + } + + killAllBookies(lh1, newBkAddr); + + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh1, 0, 9); + verifyRecoveredLedgers(lh2, 0, 9); + } finally { + rw.shutdown(); + } + + } + + /** + * Tests that ReplicationWorker should fence the ledger and release ledger + * lock after timeout. Then replication should happen normally. + */ + @Test + public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsUR() + throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKill); + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + + // set to 3s instead of default 30s + baseConf.setOpenLedgerRereplicationGracePeriod("3000"); + ReplicationWorker rw = new ReplicationWorker(baseConf); + + @Cleanup MetadataClientDriver clientDriver = MetadataDrivers.getClientDriver( + URI.create(baseClientConf.getMetadataServiceUri())); + clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty()); + + LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory + .newLedgerUnderreplicationManager(); + rw.start(); + try { + + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + killAllBookies(lh, newBkAddr); + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + lh = bkc.openLedgerNoRecovery(lh.getId(), + BookKeeper.DigestType.CRC32, TESTPASSWD); + assertFalse("Ledger must have been closed by RW", ClientUtil + .isLedgerOpen(lh)); + } finally { + rw.shutdown(); + underReplicationManager.close(); + } + + } + + @Test + public void testBookiesNotAvailableScenarioForReplicationWorker() throws Exception { + int ensembleSize = 3; + LedgerHandle lh = bkc.createLedger(ensembleSize, ensembleSize, BookKeeper.DigestType.CRC32, TESTPASSWD); + + int numOfEntries = 7; + for (int i = 0; i < numOfEntries; i++) { + lh.addEntry(data); + } + lh.close(); + + BookieId[] bookiesKilled = new BookieId[ensembleSize]; + ServerConfiguration[] killedBookiesConfig = new ServerConfiguration[ensembleSize]; + + // kill all bookies + for (int i = 0; i < ensembleSize; i++) { + bookiesKilled[i] = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(i); + killedBookiesConfig[i] = getBkConf(bookiesKilled[i]); + LOG.info("Killing Bookie : {}", bookiesKilled[i]); + killBookie(bookiesKilled[i]); + } + + // start new bookiesToKill number of bookies + for (int i = 0; i < ensembleSize; i++) { + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + } + + // create couple of replicationworkers + ServerConfiguration newRWConf = new ServerConfiguration(baseConf); + newRWConf.setLockReleaseOfFailedLedgerGracePeriod("64"); + ReplicationWorker rw1 = new ReplicationWorker(newRWConf); + ReplicationWorker rw2 = new ReplicationWorker(newRWConf); + + @Cleanup + MetadataClientDriver clientDriver = MetadataDrivers + .getClientDriver(URI.create(baseClientConf.getMetadataServiceUri())); + clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty()); + + LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + try { + //mark ledger underreplicated + for (int i = 0; i < bookiesKilled.length; i++) { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), bookiesKilled[i].toString()); + } + while (!ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) { + Thread.sleep(100); + } + rw1.start(); + rw2.start(); + + AtomicBoolean isBookieRestarted = new AtomicBoolean(false); + + (new Thread(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(3000); + isBookieRestarted.set(true); + /* + * after sleeping for 3000 msecs, restart one of the + * bookie, so that replication can succeed. + */ + startBookie(killedBookiesConfig[0]); + } catch (Exception e) { + e.printStackTrace(); + } + } + })).start(); + + int rw1PrevFailedAttemptsCount = 0; + int rw2PrevFailedAttemptsCount = 0; + while (!isBookieRestarted.get()) { + /* + * since all the bookies containing the ledger entries are down + * replication wouldnt have succeeded. + */ + assertTrue("Ledger: " + lh.getId() + " should be underreplicated", + ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)); + + // the number of failed attempts should have increased. + int rw1CurFailedAttemptsCount = rw1.replicationFailedLedgers.get(lh.getId()).get(); + assertTrue( + "The current number of failed attempts: " + rw1CurFailedAttemptsCount + + " should be greater than or equal to previous value: " + rw1PrevFailedAttemptsCount, + rw1CurFailedAttemptsCount >= rw1PrevFailedAttemptsCount); + rw1PrevFailedAttemptsCount = rw1CurFailedAttemptsCount; + + int rw2CurFailedAttemptsCount = rw2.replicationFailedLedgers.get(lh.getId()).get(); + assertTrue( + "The current number of failed attempts: " + rw2CurFailedAttemptsCount + + " should be greater than or equal to previous value: " + rw2PrevFailedAttemptsCount, + rw2CurFailedAttemptsCount >= rw2PrevFailedAttemptsCount); + rw2PrevFailedAttemptsCount = rw2CurFailedAttemptsCount; + + Thread.sleep(50); + } + + /** + * since one of the killed bookie is restarted, replicationworker + * should succeed in replicating this under replicated ledger and it + * shouldn't be under replicated anymore. + */ + int timeToWaitForReplicationToComplete = 20000; + int timeWaited = 0; + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) { + Thread.sleep(100); + timeWaited += 100; + if (timeWaited == timeToWaitForReplicationToComplete) { + fail("Ledger should be replicated by now"); + } + } + + rw1PrevFailedAttemptsCount = rw1.replicationFailedLedgers.get(lh.getId()).get(); + rw2PrevFailedAttemptsCount = rw2.replicationFailedLedgers.get(lh.getId()).get(); + Thread.sleep(2000); + // now since the ledger is replicated, number of failed attempts + // counter shouldn't be increased even after sleeping for sometime. + assertEquals("rw1 failedattempts", rw1PrevFailedAttemptsCount, + rw1.replicationFailedLedgers.get(lh.getId()).get()); + assertEquals("rw2 failed attempts ", rw2PrevFailedAttemptsCount, + rw2.replicationFailedLedgers.get(lh.getId()).get()); + + /* + * Since these entries are eventually available, and replication has + * eventually succeeded, in one of the RW + * unableToReadEntriesForReplication should be 0. + */ + int rw1UnableToReadEntriesForReplication = rw1.unableToReadEntriesForReplication.get(lh.getId()).size(); + int rw2UnableToReadEntriesForReplication = rw2.unableToReadEntriesForReplication.get(lh.getId()).size(); + assertTrue( + "unableToReadEntriesForReplication in RW1: " + rw1UnableToReadEntriesForReplication + + " in RW2: " + + rw2UnableToReadEntriesForReplication, + (rw1UnableToReadEntriesForReplication == 0) + || (rw2UnableToReadEntriesForReplication == 0)); + } finally { + rw1.shutdown(); + rw2.shutdown(); + underReplicationManager.close(); + } + } + + class InjectedReplicationWorker extends ReplicationWorker { + CopyOnWriteArrayList delayReplicationPeriods; + + public InjectedReplicationWorker(ServerConfiguration conf, StatsLogger statsLogger, + CopyOnWriteArrayList delayReplicationPeriods) + throws CompatibilityException, ReplicationException.UnavailableException, + InterruptedException, IOException { + super(conf, statsLogger); + this.delayReplicationPeriods = delayReplicationPeriods; + } + + @Override + protected void scheduleTaskWithDelay(TimerTask timerTask, long delayPeriod) { + delayReplicationPeriods.add(delayPeriod); + super.scheduleTaskWithDelay(timerTask, delayPeriod); + } + } + + @Test + public void testDeferLedgerLockReleaseForReplicationWorker() throws Exception { + int ensembleSize = 3; + LedgerHandle lh = bkc.createLedger(ensembleSize, ensembleSize, BookKeeper.DigestType.CRC32, TESTPASSWD); + int numOfEntries = 7; + for (int i = 0; i < numOfEntries; i++) { + lh.addEntry(data); + } + lh.close(); + + BookieId[] bookiesKilled = new BookieId[ensembleSize]; + ServerConfiguration[] killedBookiesConfig = new ServerConfiguration[ensembleSize]; + + // kill all bookies + for (int i = 0; i < ensembleSize; i++) { + bookiesKilled[i] = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(i); + killedBookiesConfig[i] = getBkConf(bookiesKilled[i]); + LOG.info("Killing Bookie : {}", bookiesKilled[i]); + killBookie(bookiesKilled[i]); + } + + // start new bookiesToKill number of bookies + for (int i = 0; i < ensembleSize; i++) { + startNewBookieAndReturnBookieId(); + } + + // create couple of replicationworkers + long lockReleaseOfFailedLedgerGracePeriod = 64L; + long baseBackoffForLockReleaseOfFailedLedger = lockReleaseOfFailedLedgerGracePeriod + / (int) Math.pow(2, ReplicationWorker.NUM_OF_EXPONENTIAL_BACKOFF_RETRIALS); + ServerConfiguration newRWConf = new ServerConfiguration(baseConf); + newRWConf.setLockReleaseOfFailedLedgerGracePeriod(Long.toString(lockReleaseOfFailedLedgerGracePeriod)); + newRWConf.setRereplicationEntryBatchSize(1000); + CopyOnWriteArrayList rw1DelayReplicationPeriods = new CopyOnWriteArrayList(); + CopyOnWriteArrayList rw2DelayReplicationPeriods = new CopyOnWriteArrayList(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger1 = statsProvider.getStatsLogger("rw1"); + TestStatsLogger statsLogger2 = statsProvider.getStatsLogger("rw2"); + ReplicationWorker rw1 = new InjectedReplicationWorker(newRWConf, statsLogger1, rw1DelayReplicationPeriods); + ReplicationWorker rw2 = new InjectedReplicationWorker(newRWConf, statsLogger2, rw2DelayReplicationPeriods); + + Counter numEntriesUnableToReadForReplication1 = statsLogger1 + .getCounter(NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION); + Counter numEntriesUnableToReadForReplication2 = statsLogger2 + .getCounter(NUM_ENTRIES_UNABLE_TO_READ_FOR_REPLICATION); + @Cleanup + MetadataClientDriver clientDriver = MetadataDrivers + .getClientDriver(URI.create(baseClientConf.getMetadataServiceUri())); + clientDriver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty()); + + LedgerManagerFactory mFactory = clientDriver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); + try { + // mark ledger underreplicated + for (int i = 0; i < bookiesKilled.length; i++) { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), bookiesKilled[i].toString()); + } + while (!ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) { + Thread.sleep(100); + } + rw1.start(); + rw2.start(); + + // wait for RWs to complete 'numOfAttemptsToWaitFor' failed attempts + int numOfAttemptsToWaitFor = 10; + while ((rw1.replicationFailedLedgers.get(lh.getId()).get() < numOfAttemptsToWaitFor) + || rw2.replicationFailedLedgers.get(lh.getId()).get() < numOfAttemptsToWaitFor) { + Thread.sleep(500); + } + + /* + * since all the bookies containing the ledger entries are down + * replication wouldn't have succeeded. + */ + assertTrue("Ledger: " + lh.getId() + " should be underreplicated", + ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)); + + /* + * since RW failed 'numOfAttemptsToWaitFor' number of times, we + * should have atleast (numOfAttemptsToWaitFor - 1) + * delayReplicationPeriods and their value should be + * (lockReleaseOfFailedLedgerGracePeriod/16) , 2 * previous value,.. + * with max : lockReleaseOfFailedLedgerGracePeriod + */ + for (int i = 0; i < ((numOfAttemptsToWaitFor - 1)); i++) { + long expectedDelayValue = Math.min(lockReleaseOfFailedLedgerGracePeriod, + baseBackoffForLockReleaseOfFailedLedger * (1 << i)); + assertEquals("RW1 delayperiod", (Long) expectedDelayValue, rw1DelayReplicationPeriods.get(i)); + assertEquals("RW2 delayperiod", (Long) expectedDelayValue, rw2DelayReplicationPeriods.get(i)); + } + + /* + * RW wont try to replicate until and unless RW succeed in reading + * those failed entries before proceeding with replication of under + * replicated fragment, so the numEntriesUnableToReadForReplication + * should be just 'numOfEntries', though RW failed to replicate + * multiple times. + */ + assertEquals("numEntriesUnableToReadForReplication for RW1", Long.valueOf((long) numOfEntries), + numEntriesUnableToReadForReplication1.get()); + assertEquals("numEntriesUnableToReadForReplication for RW2", Long.valueOf((long) numOfEntries), + numEntriesUnableToReadForReplication2.get()); + + /* + * Since these entries are unavailable, + * unableToReadEntriesForReplication should be of size numOfEntries. + */ + assertEquals("RW1 unabletoreadentries", numOfEntries, + rw1.unableToReadEntriesForReplication.get(lh.getId()).size()); + assertEquals("RW2 unabletoreadentries", numOfEntries, + rw2.unableToReadEntriesForReplication.get(lh.getId()).size()); + } finally { + rw1.shutdown(); + rw2.shutdown(); + underReplicationManager.close(); + } + } + + /** + * Tests that ReplicationWorker should not have identified for postponing + * the replication if ledger is in open state and lastFragment is not in + * underReplication state. Note that RW should not fence such ledgers. + */ + @Test + public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR() + throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, + TESTPASSWD); + + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + BookieId replicaToKill = lh.getLedgerMetadata().getAllEnsembles().get(0L).get(0); + + LOG.info("Killing Bookie : {}", replicaToKill); + killBookie(replicaToKill); + + BookieId newBkAddr = startNewBookieAndReturnBookieId(); + LOG.info("New Bookie addr : {}", newBkAddr); + + // Reform ensemble...Making sure that last fragment is not in + // under-replication + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + + ReplicationWorker rw = new ReplicationWorker(baseConf); + + baseClientConf.setMetadataServiceUri(zkUtil.getMetadataServiceUri()); + + @Cleanup MetadataClientDriver driver = MetadataDrivers.getClientDriver( + URI.create(baseClientConf.getMetadataServiceUri())); + driver.initialize(baseClientConf, scheduler, NullStatsLogger.INSTANCE, Optional.empty()); + + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + + LedgerUnderreplicationManager underReplicationManager = mFactory + .newLedgerUnderreplicationManager(); + + rw.start(); + try { + + underReplicationManager.markLedgerUnderreplicated(lh.getId(), + replicaToKill.toString()); + while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh + .getId(), basePath)) { + Thread.sleep(100); + } + + killAllBookies(lh, newBkAddr); + + // Should be able to read the entries from 0-9 + verifyRecoveredLedgers(lh, 0, 9); + lh = bkc.openLedgerNoRecovery(lh.getId(), + BookKeeper.DigestType.CRC32, TESTPASSWD); + + // Ledger should be still in open state + assertTrue("Ledger must have been closed by RW", ClientUtil + .isLedgerOpen(lh)); + } finally { + rw.shutdown(); + underReplicationManager.close(); + } + + } + + /** + * Test that the replication worker will not shutdown on a simple ZK disconnection. + */ + @Test + public void testRWZKConnectionLost() throws Exception { + try (ZooKeeperClient zk = ZooKeeperClient.newBuilder() + .connectString(zkUtil.getZooKeeperConnectString()) + .sessionTimeoutMs(10000) + .build()) { + + ReplicationWorker rw = new ReplicationWorker(baseConf); + rw.start(); + for (int i = 0; i < 10; i++) { + if (rw.isRunning()) { + break; + } + Thread.sleep(1000); + } + assertTrue("Replication worker should be running", rw.isRunning()); + + stopZKCluster(); + // ZK is down for shorter period than reconnect timeout + Thread.sleep(1000); + startZKCluster(); + + assertTrue("Replication worker should not shutdown", rw.isRunning()); + } + } + + /** + * Test that the replication worker shuts down on non-recoverable ZK connection loss. + */ + @Test + public void testRWZKConnectionLostOnNonRecoverableZkError() throws Exception { + for (int j = 0; j < 3; j++) { + LedgerHandle lh = bkc.createLedger(1, 1, 1, + BookKeeper.DigestType.CRC32, TESTPASSWD, + null); + final long createdLedgerId = lh.getId(); + for (int i = 0; i < 10; i++) { + lh.addEntry(data); + } + lh.close(); + } + + killBookie(2); + killBookie(1); + startNewBookie(); + startNewBookie(); + + servers.get(0).getConfiguration().setRwRereplicateBackoffMs(100); + servers.get(0).startAutoRecovery(); + + Auditor auditor = getAuditor(10, TimeUnit.SECONDS); + ReplicationWorker rw = servers.get(0).getReplicationWorker(); + + ZkLedgerUnderreplicationManager ledgerUnderreplicationManager = + (ZkLedgerUnderreplicationManager) FieldUtils.readField(auditor, + "ledgerUnderreplicationManager", true); + + ZooKeeper zkc = (ZooKeeper) FieldUtils.readField(ledgerUnderreplicationManager, "zkc", true); + auditor.submitAuditTask().get(); + + assertTrue(zkc.getState().isConnected()); + zkc.close(); + assertFalse(zkc.getState().isConnected()); + + auditor.submitAuditTask(); + rw.run(); + + for (int i = 0; i < 10; i++) { + if (!rw.isRunning() && !auditor.isRunning()) { + break; + } + Thread.sleep(1000); + } + assertFalse("Replication worker should NOT be running", rw.isRunning()); + assertFalse("Auditor should NOT be running", auditor.isRunning()); + } + + private void killAllBookies(LedgerHandle lh, BookieId excludeBK) + throws Exception { + // Killing all bookies except newly replicated bookie + for (Entry> entry : + lh.getLedgerMetadata().getAllEnsembles().entrySet()) { + List bookies = entry.getValue(); + for (BookieId bookie : bookies) { + if (bookie.equals(excludeBK)) { + continue; + } + killBookie(bookie); + } + } + } + + private void verifyRecoveredLedgers(LedgerHandle lh, long startEntryId, + long endEntryId) throws BKException, InterruptedException { + LedgerHandle lhs = bkc.openLedgerNoRecovery(lh.getId(), + BookKeeper.DigestType.CRC32, TESTPASSWD); + Enumeration entries = lhs.readEntries(startEntryId, + endEntryId); + assertTrue("Should have the elements", entries.hasMoreElements()); + while (entries.hasMoreElements()) { + LedgerEntry entry = entries.nextElement(); + assertEquals("TestReplicationWorker", new String(entry.getEntry())); + } + } + + @Test + public void testReplicateEmptyOpenStateLedger() throws Exception { + LedgerHandle lh = bkc.createLedger(3, 3, 2, BookKeeper.DigestType.CRC32, TESTPASSWD); + assertFalse(lh.getLedgerMetadata().isClosed()); + + List firstEnsemble = lh.getLedgerMetadata().getAllEnsembles().firstEntry().getValue(); + List ensemble = lh.getLedgerMetadata().getAllEnsembles().entrySet().iterator().next().getValue(); + killBookie(ensemble.get(1)); + + startNewBookie(); + baseConf.setOpenLedgerRereplicationGracePeriod(String.valueOf(30)); + ReplicationWorker replicationWorker = new ReplicationWorker(baseConf); + replicationWorker.start(); + + try { + underReplicationManager.markLedgerUnderreplicated(lh.getId(), ensemble.get(1).toString()); + Awaitility.waitAtMost(60, TimeUnit.SECONDS).untilAsserted(() -> + assertFalse(ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)) + ); + + LedgerHandle lh1 = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD); + assertTrue(lh1.getLedgerMetadata().isClosed()); + } finally { + replicationWorker.shutdown(); + } + } + + @Test + public void testRepairedNotAdheringPlacementPolicyLedgerFragmentsOnRack() throws Exception { + testRepairedNotAdheringPlacementPolicyLedgerFragments(RackawareEnsemblePlacementPolicy.class, null); + } + + @Test + public void testReplicationStats() throws Exception { + BiConsumer checkReplicationStats = (first, rw) -> { + try { + final Method rereplicate = rw.getClass().getDeclaredMethod("rereplicate"); + rereplicate.setAccessible(true); + final Object result = rereplicate.invoke(rw); + final Field statsLoggerField = rw.getClass().getDeclaredField("statsLogger"); + statsLoggerField.setAccessible(true); + final TestStatsLogger statsLogger = (TestStatsLogger) statsLoggerField.get(rw); + + final Counter numDeferLedgerLockReleaseOfFailedLedgerCounter = + statsLogger.getCounter(ReplicationStats.NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER); + final Counter numLedgersReplicatedCounter = + statsLogger.getCounter(ReplicationStats.NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED); + final Counter numNotAdheringPlacementLedgersCounter = statsLogger + .getCounter(ReplicationStats.NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED); + + assertEquals("NUM_DEFER_LEDGER_LOCK_RELEASE_OF_FAILED_LEDGER", + 1, numDeferLedgerLockReleaseOfFailedLedgerCounter.get().longValue()); + + if (first) { + assertFalse((boolean) result); + assertEquals("NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED", + 0, numLedgersReplicatedCounter.get().longValue()); + assertEquals("NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED", + 0, numNotAdheringPlacementLedgersCounter.get().longValue()); + + } else { + assertTrue((boolean) result); + assertEquals("NUM_FULL_OR_PARTIAL_LEDGERS_REPLICATED", + 1, numLedgersReplicatedCounter.get().longValue()); + assertEquals("NUM_NOT_ADHERING_PLACEMENT_LEDGERS_REPLICATED", + 1, numNotAdheringPlacementLedgersCounter.get().longValue()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + testRepairedNotAdheringPlacementPolicyLedgerFragments( + RackawareEnsemblePlacementPolicy.class, checkReplicationStats); + } + + private void testRepairedNotAdheringPlacementPolicyLedgerFragments( + Class placementPolicyClass, + BiConsumer checkReplicationStats) throws Exception { + List firstThreeBookies = servers.stream().map(ele -> { + try { + return ele.getServer().getBookieId(); + } catch (UnknownHostException e) { + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + + baseClientConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); + baseClientConf.setProperty("enforceStrictZoneawarePlacement", false); + bkc.close(); + bkc = new BookKeeperTestClient(baseClientConf) { + @Override + protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, + DNSToSwitchMapping dnsResolver, + HashedWheelTimer timer, + FeatureProvider featureProvider, + StatsLogger statsLogger, + BookieAddressResolver bookieAddressResolver) + throws IOException { + EnsemblePlacementPolicy ensemblePlacementPolicy = null; + if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies); + } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies); + } + ensemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, + featureProvider, statsLogger, bookieAddressResolver); + return ensemblePlacementPolicy; + } + }; + + //This ledger not adhering placement policy, the combine(0,1,2) rack is 1. + LedgerHandle lh = bkc.createLedger(3, 3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); + + int entrySize = 10; + for (int i = 0; i < entrySize; i++) { + lh.addEntry(data); + } + lh.close(); + + int minNumRacksPerWriteQuorumConfValue = 2; + + ServerConfiguration servConf = new ServerConfiguration(confByIndex(0)); + servConf.setMinNumRacksPerWriteQuorum(minNumRacksPerWriteQuorumConfValue); + servConf.setProperty("reppDnsResolverClass", StaticDNSResolver.class.getName()); + servConf.setAuditorPeriodicPlacementPolicyCheckInterval(1000); + servConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + + MutableObject auditorRef = new MutableObject(); + try { + TestStatsLogger statsLogger = startAuditorAndWaitForPlacementPolicyCheck(servConf, auditorRef); + Gauge ledgersNotAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_NOT_ADHERING_TO_PLACEMENT_POLICY guage value", + 1, ledgersNotAdheringToPlacementPolicyGuage.getSample()); + Gauge ledgersSoftlyAdheringToPlacementPolicyGuage = statsLogger + .getGauge(ReplicationStats.NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY); + assertEquals("NUM_LEDGERS_SOFTLY_ADHERING_TO_PLACEMENT_POLICY guage value", + 0, ledgersSoftlyAdheringToPlacementPolicyGuage.getSample()); + } finally { + Auditor auditor = auditorRef.getValue(); + if (auditor != null) { + auditor.close(); + } + } + + ZooKeeper zk = getZk((PulsarMetadataClientDriver) bkc.getMetadataClientDriver()); + + + Stat stat = zk.exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false); + assertNotNull(stat); + + baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(true); + BookKeeper bookKeeper = new BookKeeperTestClient(baseClientConf) { + @Override + protected EnsemblePlacementPolicy initializeEnsemblePlacementPolicy(ClientConfiguration conf, + DNSToSwitchMapping dnsResolver, + HashedWheelTimer timer, + FeatureProvider featureProvider, + StatsLogger statsLogger, + BookieAddressResolver bookieAddressResolver) + throws IOException { + EnsemblePlacementPolicy ensemblePlacementPolicy = null; + if (ZoneawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildZoneAwareEnsemblePlacementPolicy(firstThreeBookies); + } else if (RackawareEnsemblePlacementPolicy.class == placementPolicyClass) { + ensemblePlacementPolicy = buildRackAwareEnsemblePlacementPolicy(firstThreeBookies); + } + ensemblePlacementPolicy.initialize(conf, Optional.ofNullable(dnsResolver), timer, + featureProvider, statsLogger, bookieAddressResolver); + return ensemblePlacementPolicy; + } + }; + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(REPLICATION_SCOPE); + ReplicationWorker rw = new ReplicationWorker(baseConf, bookKeeper, false, statsLogger); + + if (checkReplicationStats != null) { + checkReplicationStats.accept(true, rw); + } else { + rw.start(); + } + + //start new bookie, the rack is /rack2 + BookieId newBookieId = startNewBookieAndReturnBookieId(); + + if (checkReplicationStats != null) { + checkReplicationStats.accept(false, rw); + } + + Awaitility.await().untilAsserted(() -> { + LedgerMetadata metadata = bkc.getLedgerManager().readLedgerMetadata(lh.getId()).get().getValue(); + List newBookies = metadata.getAllEnsembles().get(0L); + assertTrue(newBookies.contains(newBookieId)); + }); + + Awaitility.await().untilAsserted(() -> { + Stat stat1 = zk.exists("/ledgers/underreplication/ledgers/0000/0000/0000/0000/urL0000000000", false); + assertNull(stat1); + }); + + for (BookieId rack1Book : firstThreeBookies) { + killBookie(rack1Book); + } + + verifyRecoveredLedgers(lh, 0, entrySize - 1); + + if (checkReplicationStats == null) { + rw.shutdown(); + } + baseConf.setRepairedPlacementPolicyNotAdheringBookieEnable(false); + bookKeeper.close(); + } + + private EnsemblePlacementPolicy buildRackAwareEnsemblePlacementPolicy(List bookieIds) { + return new RackawareEnsemblePlacementPolicy() { + @Override + public String resolveNetworkLocation(BookieId addr) { + if (bookieIds.contains(addr)) { + return "/rack1"; + } + //The other bookie is /rack2 + return "/rack2"; + } + }; + } + + private EnsemblePlacementPolicy buildZoneAwareEnsemblePlacementPolicy(List firstThreeBookies) { + return new ZoneawareEnsemblePlacementPolicy() { + @Override + protected String resolveNetworkLocation(BookieId addr) { + //The first three bookie 1 is /zone1/ud1 + //The first three bookie 2,3 is /zone1/ud2 + if (firstThreeBookies.get(0).equals(addr)) { + return "/zone1/ud1"; + } else if (firstThreeBookies.contains(addr)) { + return "/zone1/ud2"; + } + //The other bookie is /zone2/ud1 + return "/zone2/ud1"; + } + }; + } + + private TestStatsLogger startAuditorAndWaitForPlacementPolicyCheck(ServerConfiguration servConf, + MutableObject auditorRef) + throws MetadataException, CompatibilityException, KeeperException, + InterruptedException, ReplicationException.UnavailableException, UnknownHostException { + LedgerManagerFactory mFactory = driver.getLedgerManagerFactory(); + LedgerUnderreplicationManager urm = mFactory.newLedgerUnderreplicationManager(); + TestStatsProvider statsProvider = new TestStatsProvider(); + TestStatsLogger statsLogger = statsProvider.getStatsLogger(AUDITOR_SCOPE); + TestStatsProvider.TestOpStatsLogger placementPolicyCheckStatsLogger = + (TestStatsProvider.TestOpStatsLogger) statsLogger + .getOpStatsLogger(ReplicationStats.PLACEMENT_POLICY_CHECK_TIME); + + final AuditorPeriodicCheckTest.TestAuditor auditor = new AuditorPeriodicCheckTest.TestAuditor( + BookieImpl.getBookieId(servConf).toString(), servConf, bkc, false, statsLogger, null); + auditorRef.setValue(auditor); + CountDownLatch latch = auditor.getLatch(); + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 0, + placementPolicyCheckStatsLogger.getSuccessCount()); + urm.setPlacementPolicyCheckCTime(-1); + auditor.start(); + /* + * since placementPolicyCheckCTime is set to -1, placementPolicyCheck should be + * scheduled to run with no initialdelay + */ + assertTrue("placementPolicyCheck should have executed", latch.await(20, TimeUnit.SECONDS)); + for (int i = 0; i < 20; i++) { + Thread.sleep(100); + if (placementPolicyCheckStatsLogger.getSuccessCount() >= 1) { + break; + } + } + assertEquals("PLACEMENT_POLICY_CHECK_TIME SuccessCount", 1, + placementPolicyCheckStatsLogger.getSuccessCount()); + return statsLogger; + } + + private ZooKeeper getZk(PulsarMetadataClientDriver pulsarMetadataClientDriver) throws Exception { + PulsarLedgerManagerFactory pulsarLedgerManagerFactory = + (PulsarLedgerManagerFactory) pulsarMetadataClientDriver.getLedgerManagerFactory(); + Field field = pulsarLedgerManagerFactory.getClass().getDeclaredField("store"); + field.setAccessible(true); + ZKMetadataStore zkMetadataStore = (ZKMetadataStore) field.get(pulsarLedgerManagerFactory); + return zkMetadataStore.getZkClient(); + } +} From 01dfd3e03eab64a17d2d925448f3b571ce710d90 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 7 Oct 2023 23:45:30 +0800 Subject: [PATCH 307/494] Revert "[fix][broker] Fix inconsistent topic policy (#21231)" This reverts commit a0e2104642c5081986dbe7847c71997b06030eaa. --- .../ProxySaslAuthenticationTest.java | 13 +- .../authentication/SaslAuthenticateTest.java | 11 +- .../pulsar/broker/service/BrokerService.java | 295 +++++++++--------- .../SystemTopicBasedTopicPoliciesService.java | 100 ++---- .../broker/service/TopicPoliciesService.java | 41 --- .../broker/admin/PersistentTopicsTest.java | 1 - .../broker/admin/TopicPoliciesTest.java | 2 +- .../TopicPoliciesWithBrokerRestartTest.java | 104 ------ .../pulsar/broker/admin/TopicsAuthTest.java | 2 - .../pulsar/broker/auth/AuthLogsTest.java | 2 - .../broker/auth/MockAuthentication.java | 6 +- .../auth/MockedPulsarServiceBaseTest.java | 19 -- .../service/BrokerBookieIsolationTest.java | 3 +- ...temTopicBasedTopicPoliciesServiceTest.java | 6 +- .../persistent/PersistentTopicTest.java | 3 +- ...enticationTlsHostnameVerificationTest.java | 3 - .../AuthorizationProducerConsumerTest.java | 5 - .../client/api/MutualAuthenticationTest.java | 2 +- ...okenAuthenticatedProducerConsumerTest.java | 3 - ...uth2AuthenticatedProducerConsumerTest.java | 14 +- ...reTlsProducerConsumerTestWithAuthTest.java | 22 -- .../PatternTopicsConsumerImplAuthTest.java | 1 - .../standalone_no_client_auth.conf | 3 +- .../proxy/server/ProxyAuthenticationTest.java | 2 +- .../server/ProxyForwardAuthDataTest.java | 2 +- .../server/ProxyRolesEnforcementTest.java | 2 +- .../server/ProxyWithAuthorizationNegTest.java | 2 - .../pulsar/sql/presto/TestPulsarAuth.java | 4 - .../ExtensibleLoadManagerTest.java | 1 - .../integration/presto/TestPulsarSQLAuth.java | 1 - 30 files changed, 197 insertions(+), 478 deletions(-) delete mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java index f0e45aa734afb..261efe680f862 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java @@ -49,7 +49,6 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.auth.AuthenticationSasl; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; -import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.proxy.server.ProxyConfiguration; import org.apache.pulsar.proxy.server.ProxyService; import org.slf4j.Logger; @@ -194,17 +193,15 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setClusterName("test"); conf.setSuperUserRoles(ImmutableSet.of("client/" + localHostname + "@" + kdc.getRealm())); - // set admin auth, to verify admin web resources - Map clientSaslConfig = new HashMap<>(); - clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); - clientSaslConfig.put("serverType", "broker"); - conf.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); - conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory - .getMapper().getObjectMapper().writeValueAsString(clientSaslConfig)); super.init(); lookupUrl = new URI(pulsar.getBrokerServiceUrl()); + + // set admin auth, to verify admin web resources + Map clientSaslConfig = new HashMap<>(); + clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); + clientSaslConfig.put("serverType", "broker"); log.info("set client jaas section name: PulsarClient"); admin = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl.toString()) diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java index 76c6f023d9a36..8a0d0392d1333 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java @@ -53,7 +53,6 @@ import org.apache.pulsar.client.impl.auth.AuthenticationSasl; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.sasl.SaslConstants; -import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; @@ -181,12 +180,7 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setClusterName("test"); conf.setSuperUserRoles(ImmutableSet.of("client" + "@" + kdc.getRealm())); - Map clientSaslConfig = new HashMap<>(); - clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); - clientSaslConfig.put("serverType", "broker"); - conf.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); - conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory - .getMapper().getObjectMapper().writeValueAsString(clientSaslConfig)); + super.init(); lookupUrl = new URI(pulsar.getWebServiceAddress()); @@ -197,6 +191,9 @@ protected void setup() throws Exception { .authentication(authSasl)); // set admin auth, to verify admin web resources + Map clientSaslConfig = new HashMap<>(); + clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); + clientSaslConfig.put("serverType", "broker"); log.info("set client jaas section name: PulsarClient"); admin = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl.toString()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 7c7dec864e2e8..d6b17f4faa4da 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1769,172 +1769,165 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { }); } - public CompletableFuture getManagedLedgerConfig(@Nonnull TopicName topicName) { - requireNonNull(topicName); + public CompletableFuture getManagedLedgerConfig(TopicName topicName) { NamespaceName namespace = topicName.getNamespaceObject(); ServiceConfiguration serviceConfig = pulsar.getConfiguration(); NamespaceResources nsr = pulsar.getPulsarResources().getNamespaceResources(); LocalPoliciesResources lpr = pulsar.getPulsarResources().getLocalPolicies(); - final CompletableFuture> topicPoliciesFuture; - if (pulsar.getConfig().isTopicLevelPoliciesEnabled() - && !NamespaceService.isSystemServiceNamespace(namespace.toString()) - && !SystemTopicNames.isTopicPoliciesSystemTopic(topicName.toString())) { - topicPoliciesFuture = pulsar.getTopicPoliciesService().getTopicPoliciesAsync(topicName); - } else { - topicPoliciesFuture = CompletableFuture.completedFuture(Optional.empty()); - } - return topicPoliciesFuture.thenCompose(topicPoliciesOptional -> { - final CompletableFuture> nsPolicies = nsr.getPoliciesAsync(namespace); - final CompletableFuture> lcPolicies = lpr.getLocalPoliciesAsync(namespace); - return nsPolicies.thenCombine(lcPolicies, (policies, localPolicies) -> { - PersistencePolicies persistencePolicies = null; - RetentionPolicies retentionPolicies = null; - OffloadPoliciesImpl topicLevelOffloadPolicies = null; - if (topicPoliciesOptional.isPresent()) { - final TopicPolicies topicPolicies = topicPoliciesOptional.get(); - persistencePolicies = topicPolicies.getPersistence(); - retentionPolicies = topicPolicies.getRetentionPolicies(); - topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies(); - } - - if (persistencePolicies == null) { - persistencePolicies = policies.map(p -> p.persistence).orElseGet( - () -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), - serviceConfig.getManagedLedgerDefaultWriteQuorum(), - serviceConfig.getManagedLedgerDefaultAckQuorum(), - serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit())); - } + return nsr.getPoliciesAsync(namespace) + .thenCombine(lpr.getLocalPoliciesAsync(namespace), (policies, localPolicies) -> { + PersistencePolicies persistencePolicies = null; + RetentionPolicies retentionPolicies = null; + OffloadPoliciesImpl topicLevelOffloadPolicies = null; + + if (pulsar.getConfig().isTopicLevelPoliciesEnabled() + && !NamespaceService.isSystemServiceNamespace(namespace.toString())) { + final TopicPolicies topicPolicies = pulsar.getTopicPoliciesService() + .getTopicPoliciesIfExists(topicName); + if (topicPolicies != null) { + persistencePolicies = topicPolicies.getPersistence(); + retentionPolicies = topicPolicies.getRetentionPolicies(); + topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies(); + } + } - if (retentionPolicies == null) { - retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( - () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), - serviceConfig.getDefaultRetentionSizeInMB()) - ); - } + if (persistencePolicies == null) { + persistencePolicies = policies.map(p -> p.persistence).orElseGet( + () -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), + serviceConfig.getManagedLedgerDefaultWriteQuorum(), + serviceConfig.getManagedLedgerDefaultAckQuorum(), + serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit())); + } - ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble()); - managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum()); - managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum()); - - if (serviceConfig.isStrictBookieAffinityEnabled()) { - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( - IsolatedBookieEnsemblePlacementPolicy.class); - if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } else if (isSystemTopic(topicName)) { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); - properties.put(IsolatedBookieEnsemblePlacementPolicy - .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } else { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + if (retentionPolicies == null) { + retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( + () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), + serviceConfig.getDefaultRetentionSizeInMB()) + ); } - } else { - if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { + + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble()); + managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum()); + managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum()); + + if (serviceConfig.isStrictBookieAffinityEnabled()) { managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( IsolatedBookieEnsemblePlacementPolicy.class); - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } else if (isSystemTopic(topicName)) { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); + properties.put(IsolatedBookieEnsemblePlacementPolicy + .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } else { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } + } else { + if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( + IsolatedBookieEnsemblePlacementPolicy.class); + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } } - } - managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate()); - managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType()); - managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword()); - - managedLedgerConfig - .setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist()); - managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( - serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); - managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( - serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); - managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); - managedLedgerConfig - .setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), - TimeUnit.MINUTES); - managedLedgerConfig - .setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), - TimeUnit.MINUTES); - managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes()); - - managedLedgerConfig.setMetadataOperationsTimeoutSeconds( - serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds()); - managedLedgerConfig - .setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds()); - managedLedgerConfig - .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); - managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( - serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); - managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); - managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); - managedLedgerConfig - .setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger()); - - managedLedgerConfig - .setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds()); - managedLedgerConfig - .setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); - managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); - managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData()); - managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery()); - managedLedgerConfig.setInactiveLedgerRollOverTime( - serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS); - managedLedgerConfig.setCacheEvictionByMarkDeletedPosition( - serviceConfig.isCacheEvictionByMarkDeletedPosition()); - managedLedgerConfig.setMinimumBacklogCursorsForCaching( - serviceConfig.getManagedLedgerMinimumBacklogCursorsForCaching()); - managedLedgerConfig.setMinimumBacklogEntriesForCaching( - serviceConfig.getManagedLedgerMinimumBacklogEntriesForCaching()); - managedLedgerConfig.setMaxBacklogBetweenCursorsForCaching( - serviceConfig.getManagedLedgerMaxBacklogBetweenCursorsForCaching()); - - OffloadPoliciesImpl nsLevelOffloadPolicies = - (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null); - OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration( - topicLevelOffloadPolicies, - OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), - getPulsar().getConfig().getProperties()); - if (NamespaceService.isSystemServiceNamespace(namespace.toString())) { - managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE); - } else { - if (topicLevelOffloadPolicies != null) { - try { - LedgerOffloader topicLevelLedgerOffLoader = - pulsar().createManagedLedgerOffloader(offloadPolicies); - managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); - } catch (PulsarServerException e) { - throw new RuntimeException(e); + managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate()); + managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType()); + managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword()); + + managedLedgerConfig + .setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist()); + managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( + serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); + managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( + serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); + managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); + managedLedgerConfig + .setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), + TimeUnit.MINUTES); + managedLedgerConfig + .setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), + TimeUnit.MINUTES); + managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes()); + + managedLedgerConfig.setMetadataOperationsTimeoutSeconds( + serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds()); + managedLedgerConfig + .setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds()); + managedLedgerConfig + .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); + managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); + managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( + serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); + managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); + managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); + managedLedgerConfig + .setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger()); + + managedLedgerConfig + .setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds()); + managedLedgerConfig + .setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); + managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); + managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData()); + managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery()); + managedLedgerConfig.setInactiveLedgerRollOverTime( + serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS); + managedLedgerConfig.setCacheEvictionByMarkDeletedPosition( + serviceConfig.isCacheEvictionByMarkDeletedPosition()); + managedLedgerConfig.setMinimumBacklogCursorsForCaching( + serviceConfig.getManagedLedgerMinimumBacklogCursorsForCaching()); + managedLedgerConfig.setMinimumBacklogEntriesForCaching( + serviceConfig.getManagedLedgerMinimumBacklogEntriesForCaching()); + managedLedgerConfig.setMaxBacklogBetweenCursorsForCaching( + serviceConfig.getManagedLedgerMaxBacklogBetweenCursorsForCaching()); + + OffloadPoliciesImpl nsLevelOffloadPolicies = + (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null); + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration( + topicLevelOffloadPolicies, + OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), + getPulsar().getConfig().getProperties()); + if (NamespaceService.isSystemServiceNamespace(namespace.toString())) { + managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE); + } else { + if (topicLevelOffloadPolicies != null) { + try { + LedgerOffloader topicLevelLedgerOffLoader = + pulsar().createManagedLedgerOffloader(offloadPolicies); + managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); + } catch (PulsarServerException e) { + throw new RuntimeException(e); + } + } else { + //If the topic level policy is null, use the namespace level + managedLedgerConfig + .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); } - } else { - //If the topic level policy is null, use the namespace level - managedLedgerConfig - .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); } - } - managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( - serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled()); - managedLedgerConfig.setNewEntriesCheckDelayInMillis( - serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis()); - return managedLedgerConfig; - }); - }); + managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( + serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled()); + managedLedgerConfig.setNewEntriesCheckDelayInMillis( + serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis()); + return managedLedgerConfig; + }); } private void addTopicToStatsMaps(TopicName topicName, Topic topic) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index ed76d37ae2536..09f8de818db0a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -18,14 +18,12 @@ */ package org.apache.pulsar.broker.service; -import static java.util.Objects.requireNonNull; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -56,7 +54,6 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.FutureUtil; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,8 +78,8 @@ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesServic private final Map>> readerCaches = new ConcurrentHashMap<>(); - - final Map> policyCacheInitMap = new ConcurrentHashMap<>(); + @VisibleForTesting + final Map policyCacheInitMap = new ConcurrentHashMap<>(); @VisibleForTesting final Map>> listeners = new ConcurrentHashMap<>(); @@ -222,12 +219,12 @@ public TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) throws TopicPoliciesCacheNotInitException { if (!policyCacheInitMap.containsKey(topicName.getNamespaceObject())) { NamespaceName namespace = topicName.getNamespaceObject(); - prepareInitPoliciesCacheAsync(namespace); + prepareInitPoliciesCache(namespace, new CompletableFuture<>()); } MutablePair result = new MutablePair<>(); policyCacheInitMap.compute(topicName.getNamespaceObject(), (k, initialized) -> { - if (initialized == null || !initialized.isDone()) { + if (initialized == null || !initialized) { result.setLeft(new TopicPoliciesCacheNotInitException()); } else { TopicPolicies topicPolicies = @@ -245,34 +242,6 @@ public TopicPolicies getTopicPolicies(TopicName topicName, } } - @NotNull - @Override - public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName, - boolean isGlobal) { - requireNonNull(topicName); - final CompletableFuture preparedFuture = prepareInitPoliciesCacheAsync(topicName.getNamespaceObject()); - return preparedFuture.thenApply(__ -> { - final TopicPolicies candidatePolicies = isGlobal - ? globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName())) - : policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); - return Optional.ofNullable(candidatePolicies); - }); - } - - @NotNull - @Override - public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName) { - requireNonNull(topicName); - final CompletableFuture preparedFuture = prepareInitPoliciesCacheAsync(topicName.getNamespaceObject()); - return preparedFuture.thenApply(__ -> { - final TopicPolicies localPolicies = policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); - if (localPolicies != null) { - return Optional.of(localPolicies); - } - return Optional.ofNullable(globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName()))); - }); - } - @Override public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { return policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); @@ -296,48 +265,39 @@ public CompletableFuture getTopicPoliciesBypassCacheAsync(TopicNa @Override public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { + CompletableFuture result = new CompletableFuture<>(); NamespaceName namespace = namespaceBundle.getNamespaceObject(); if (NamespaceService.isHeartbeatNamespace(namespace)) { - return CompletableFuture.completedFuture(null); + result.complete(null); + return result; } synchronized (this) { if (readerCaches.get(namespace) != null) { ownedBundlesCountPerNamespace.get(namespace).incrementAndGet(); - return CompletableFuture.completedFuture(null); + result.complete(null); } else { - return prepareInitPoliciesCacheAsync(namespace); + prepareInitPoliciesCache(namespace, result); } } + return result; } - private @Nonnull CompletableFuture prepareInitPoliciesCacheAsync(@Nonnull NamespaceName namespace) { - requireNonNull(namespace); - return policyCacheInitMap.computeIfAbsent(namespace, (k) -> { - final CompletableFuture> readerCompletableFuture = + private void prepareInitPoliciesCache(@Nonnull NamespaceName namespace, CompletableFuture result) { + if (policyCacheInitMap.putIfAbsent(namespace, false) == null) { + CompletableFuture> readerCompletableFuture = createSystemTopicClientWithRetry(namespace); readerCaches.put(namespace, readerCompletableFuture); ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); - final CompletableFuture initFuture = readerCompletableFuture - .thenCompose(reader -> { - final CompletableFuture stageFuture = new CompletableFuture<>(); - initPolicesCache(reader, stageFuture); - return stageFuture - // Read policies in background - .thenAccept(__ -> readMorePoliciesAsync(reader)); - }); - initFuture.exceptionally(ex -> { - try { - log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); - cleanCacheAndCloseReader(namespace, false); - } catch (Throwable cleanupEx) { - // Adding this catch to avoid break callback chain - log.error("[{}] Failed to cleanup reader on __change_events topic", namespace, cleanupEx); - } + readerCompletableFuture.thenAccept(reader -> { + initPolicesCache(reader, result); + result.thenRun(() -> readMorePolicies(reader)); + }).exceptionally(ex -> { + log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); + cleanCacheAndCloseReader(namespace, false); + result.completeExceptionally(ex); return null; }); - // let caller know we've got an exception. - return initFuture; - }); + } } protected CompletableFuture> createSystemTopicClientWithRetry( @@ -421,7 +381,8 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp if (log.isDebugEnabled()) { log.debug("[{}] Reach the end of the system topic.", reader.getSystemTopic().getTopicName()); } - + policyCacheInitMap.computeIfPresent( + reader.getSystemTopic().getTopicName().getNamespaceObject(), (k, v) -> true); // replay policy message policiesCache.forEach(((topicName, topicPolicies) -> { if (listeners.get(topicName) != null) { @@ -434,7 +395,6 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp } } })); - future.complete(null); } }); @@ -460,13 +420,7 @@ private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean }); } - /** - * This is an async method for the background reader to continue syncing new messages. - * - * Note: You should not do any blocking call here. because it will affect - * #{@link SystemTopicBasedTopicPoliciesService#getTopicPoliciesAsync(TopicName)} method to block loading topic. - */ - private void readMorePoliciesAsync(SystemTopicClient.Reader reader) { + private void readMorePolicies(SystemTopicClient.Reader reader) { reader.readNextAsync() .thenAccept(msg -> { refreshTopicPoliciesCache(msg); @@ -474,7 +428,7 @@ private void readMorePoliciesAsync(SystemTopicClient.Reader reader) }) .whenComplete((__, ex) -> { if (ex == null) { - readMorePoliciesAsync(reader); + readMorePolicies(reader); } else { Throwable cause = FutureUtil.unwrapCompletionException(ex); if (cause instanceof PulsarClientException.AlreadyClosedException) { @@ -483,7 +437,7 @@ private void readMorePoliciesAsync(SystemTopicClient.Reader reader) reader.getSystemTopic().getTopicName().getNamespaceObject(), false); } else { log.warn("Read more topic polices exception, read again.", ex); - readMorePoliciesAsync(reader); + readMorePolicies(reader); } } }); @@ -651,7 +605,7 @@ boolean checkReaderIsCached(NamespaceName namespaceName) { } @VisibleForTesting - public CompletableFuture getPoliciesCacheInit(NamespaceName namespaceName) { + public Boolean getPoliciesCacheInit(NamespaceName namespaceName) { return policyCacheInitMap.get(namespaceName); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index aa3a6aaeff29f..c4bcc0c39353c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -22,7 +22,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import javax.annotation.Nonnull; import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.BackoffBuilder; @@ -32,7 +31,6 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.FutureUtil; -import org.jetbrains.annotations.NotNull; /** * Topic policies service. @@ -111,32 +109,6 @@ default CompletableFuture> getTopicPoliciesAsyncWithRetr return response; } - /** - * Asynchronously retrieves topic policies. - * This triggers the Pulsar broker's internal client to load policies from the - * system topic `persistent://tenant/namespace/__change_event`. - * - * @param topicName The name of the topic. - * @param isGlobal Indicates if the policies are global. - * @return A CompletableFuture containing an Optional of TopicPolicies. - * @throws NullPointerException If the topicName is null. - */ - @Nonnull - CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName, boolean isGlobal); - - /** - * Asynchronously retrieves topic policies. - * This triggers the Pulsar broker's internal client to load policies from the - * system topic `persistent://tenant/namespace/__change_event`. - * - * NOTE: If local policies are not available, it will fallback to using topic global policies. - * @param topicName The name of the topic. - * @return A CompletableFuture containing an Optional of TopicPolicies. - * @throws NullPointerException If the topicName is null. - */ - @Nonnull - CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName); - /** * Get policies for a topic without cache async. * @param topicName topic name @@ -190,19 +162,6 @@ public TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) return null; } - @NotNull - @Override - public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName, - boolean isGlobal) { - return CompletableFuture.completedFuture(Optional.empty()); - } - - @NotNull - @Override - public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName) { - return CompletableFuture.completedFuture(Optional.empty()); - } - @Override public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { return null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 8dee90af4afb4..284e50c830286 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -126,7 +126,6 @@ public void initPersistentTopics() throws Exception { @Override @BeforeMethod protected void setup() throws Exception { - conf.setTopicLevelPoliciesEnabled(false); super.internalSetup(); persistentTopics = spy(PersistentTopics.class); persistentTopics.setServletContext(new MockServletContext()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index faf141a5d1cf4..87471f4972f8d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -180,7 +180,7 @@ public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Excep assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); //make sure namespace policy reader is fully started. Awaitility.await().untilAsserted(()-> { - assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject()).isDone()); + assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject())); }); //load the topic. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java deleted file mode 100644 index 672fc2c95f890..0000000000000 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.broker.admin; - -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; -import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; -import org.apache.pulsar.broker.service.persistent.PersistentTopic; -import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.common.policies.data.RetentionPolicies; -import org.awaitility.Awaitility; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.TimeUnit; - -@Slf4j -@Test(groups = "broker-admin") -public class TopicPoliciesWithBrokerRestartTest extends MockedPulsarServiceBaseTest { - - @Override - @BeforeClass(alwaysRun = true) - protected void setup() throws Exception { - super.internalSetup(); - setupDefaultTenantAndNamespace(); - } - - @Override - @AfterClass(alwaysRun = true) - protected void cleanup() throws Exception { - super.internalCleanup(); - } - - - @Test - public void testRetentionWithBrokerRestart() throws Exception { - final int messages = 1_000; - final int topicNum = 500; - // (1) Init topic - admin.namespaces().createNamespace("public/retention"); - final String topicName = "persistent://public/retention/retention_with_broker_restart"; - admin.topics().createNonPartitionedTopic(topicName); - for (int i = 0; i < topicNum; i++) { - final String shadowTopicNames = topicName + "_" + i; - admin.topics().createNonPartitionedTopic(shadowTopicNames); - } - // (2) Set retention - final RetentionPolicies retentionPolicies = new RetentionPolicies(20, 20); - for (int i = 0; i < topicNum; i++) { - final String shadowTopicNames = topicName + "_" + i; - admin.topicPolicies().setRetention(shadowTopicNames, retentionPolicies); - } - admin.topicPolicies().setRetention(topicName, retentionPolicies); - // (3) Send messages - @Cleanup - final Producer publisher = pulsarClient.newProducer() - .topic(topicName) - .create(); - for (int i = 0; i < messages; i++) { - publisher.send((i + "").getBytes(StandardCharsets.UTF_8)); - } - // (4) Check configuration - Awaitility.await().untilAsserted(() -> { - final PersistentTopic persistentTopic1 = (PersistentTopic) - pulsar.getBrokerService().getTopic(topicName, true).join().get(); - final ManagedLedgerImpl managedLedger1 = (ManagedLedgerImpl) persistentTopic1.getManagedLedger(); - Assert.assertEquals(managedLedger1.getConfig().getRetentionSizeInMB(), 20); - Assert.assertEquals(managedLedger1.getConfig().getRetentionTimeMillis(), - TimeUnit.MINUTES.toMillis(20)); - }); - // (5) Restart broker - restartBroker(); - // (6) Check configuration again - for (int i = 0; i < topicNum; i++) { - final String shadowTopicNames = topicName + "_" + i; - admin.lookups().lookupTopic(shadowTopicNames); - final PersistentTopic persistentTopicTmp = (PersistentTopic) - pulsar.getBrokerService().getTopic(shadowTopicNames, true).join().get(); - final ManagedLedgerImpl managedLedgerTemp = (ManagedLedgerImpl) persistentTopicTmp.getManagedLedger(); - Assert.assertEquals(managedLedgerTemp.getConfig().getRetentionSizeInMB(), 20); - Assert.assertEquals(managedLedgerTemp.getConfig().getRetentionTimeMillis(), - TimeUnit.MINUTES.toMillis(20)); - } - } -} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java index 234af7afa8d09..efd8b66d754ac 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java @@ -84,8 +84,6 @@ protected void setup() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); - conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); - conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); super.internalSetup(); PulsarAdminBuilder pulsarAdminBuilder = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java index 942a42fa7aaa1..6ffcecbeb9f8b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java @@ -60,8 +60,6 @@ public void setup() throws Exception { conf.setAuthorizationEnabled(true); conf.setAuthorizationAllowWildcardsMatching(true); conf.setSuperUserRoles(Sets.newHashSet("super")); - conf.setBrokerClientAuthenticationPlugin(MockAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("user:pass.pass"); internalSetup(); try (PulsarAdmin admin = PulsarAdmin.builder() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java index 25ac59796b02c..0b1726617f71f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java @@ -29,10 +29,7 @@ public class MockAuthentication implements Authentication { private static final Logger log = LoggerFactory.getLogger(MockAuthentication.class); - private String user; - - public MockAuthentication() { - } + private final String user; public MockAuthentication(String user) { this.user = user; @@ -70,7 +67,6 @@ public String getCommandData() { @Override public void configure(Map authParams) { - this.user = authParams.get("user"); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 0f04088e21a03..b16dc27e26586 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -42,7 +42,6 @@ import lombok.Data; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; -import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.persistent.PersistentTopic; @@ -54,8 +53,6 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.ProducerImpl; -import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; -import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -225,22 +222,6 @@ protected void doInitConf() throws Exception { protected final void init() throws Exception { doInitConf(); - // trying to config the broker internal client - if (conf.getWebServicePortTls().isPresent() - && conf.getAuthenticationProviders().contains(AuthenticationProviderTls.class.getName()) - && !conf.isTlsEnabledWithKeyStore()) { - // enabled TLS - if (conf.getBrokerClientAuthenticationPlugin() == null - || conf.getBrokerClientAuthenticationPlugin().equals(AuthenticationDisabled.class.getName())) { - conf.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); - conf.setBrokerClientAuthenticationParameters("tlsCertFile:" + BROKER_CERT_FILE_PATH - + ",tlsKeyFile:" + BROKER_KEY_FILE_PATH); - conf.setBrokerClientTlsEnabled(true); - conf.setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); - conf.setBrokerClientCertificateFilePath(BROKER_CERT_FILE_PATH); - conf.setBrokerClientKeyFilePath(BROKER_KEY_FILE_PATH); - } - } startBroker(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java index 5252407892eea..951892f4ebfbc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java @@ -304,7 +304,6 @@ public void testSetRackInfoAndAffinityGroupDuringProduce() throws Exception { bookies[3].getBookieId()); ServiceConfiguration config = new ServiceConfiguration(); - config.setTopicLevelPoliciesEnabled(false); config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config.setClusterName(cluster); config.setWebServicePort(Optional.of(0)); @@ -613,9 +612,9 @@ public void testBookieIsolationWithSecondaryGroup() throws Exception { config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); - config.setTopicLevelPoliciesEnabled(false); config.setAdvertisedAddress("localhost"); config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups); + config.setManagedLedgerDefaultEnsembleSize(2); config.setManagedLedgerDefaultWriteQuorum(2); config.setManagedLedgerDefaultAckQuorum(2); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 5b70ff996756e..31b5bcb23cd98 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -141,7 +141,7 @@ public void testGetPolicy() throws ExecutionException, InterruptedException, Top // Wait for all topic policies updated. Awaitility.await().untilAsserted(() -> Assert.assertTrue(systemTopicBasedTopicPoliciesService - .getPoliciesCacheInit(TOPIC1.getNamespaceObject()).isDone())); + .getPoliciesCacheInit(TOPIC1.getNamespaceObject()))); // Assert broker is cache all topic policies Awaitility.await().untilAsserted(() -> @@ -304,8 +304,8 @@ private void prepareData() throws PulsarAdminException { @Test public void testGetPolicyTimeout() throws Exception { SystemTopicBasedTopicPoliciesService service = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); - Awaitility.await().untilAsserted(() -> assertTrue(service.policyCacheInitMap.get(TOPIC1.getNamespaceObject()).isDone())); - service.policyCacheInitMap.put(TOPIC1.getNamespaceObject(), new CompletableFuture<>()); + Awaitility.await().untilAsserted(() -> assertTrue(service.policyCacheInitMap.get(TOPIC1.getNamespaceObject()))); + service.policyCacheInitMap.put(TOPIC1.getNamespaceObject(), false); long start = System.currentTimeMillis(); Backoff backoff = new BackoffBuilder() .setInitialTime(500, TimeUnit.MILLISECONDS) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index ac2727e33eb33..41704af0b8bb2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -45,7 +45,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -621,7 +620,7 @@ public void testCheckPersistencePolicies() throws Exception { doReturn(policiesService).when(pulsar).getTopicPoliciesService(); TopicPolicies policies = new TopicPolicies(); policies.setRetentionPolicies(retentionPolicies); - doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(policiesService).getTopicPoliciesAsync(TopicName.get(topic)); + doReturn(policies).when(policiesService).getTopicPoliciesIfExists(TopicName.get(topic)); persistentTopic.onUpdate(policies); verify(persistentTopic, times(1)).checkPersistencePolicies(); Awaitility.await().untilAsserted(() -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java index a392f150e85c7..f2631f591217b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java @@ -30,7 +30,6 @@ import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.tls.PublicSuffixMatcher; import org.apache.pulsar.common.tls.TlsHostnameVerifier; -import org.assertj.core.util.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -148,7 +147,6 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname // setup broker cert which has CN = "pulsar" different than broker's hostname="localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setAuthenticationProviders(Sets.newTreeSet(AuthenticationProviderTls.class.getName())); conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_MIM_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_MIM_SERVER_KEY_FILE_PATH); @@ -190,7 +188,6 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception { // setup broker cert which has CN = "localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); - conf.setAuthenticationProviders(Sets.newTreeSet(AuthenticationProviderTls.class.getName())); conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index ba41848bf2c23..0ce3b7df07d1f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -119,7 +119,6 @@ protected void cleanup() throws Exception { public void testProducerAndConsumerAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); - conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(TestAuthorizationProvider.class.getName()); setup(); @@ -180,7 +179,6 @@ public void testProducerAndConsumerAuthorization() throws Exception { public void testSubscriberPermission() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); - conf.setTopicLevelPoliciesEnabled(false); conf.setEnablePackagesManagement(true); conf.setPackagesManagementStorageProvider(MockedPackagesStorageProvider.class.getName()); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); @@ -371,7 +369,6 @@ public void testSubscriberPermission() throws Exception { public void testClearBacklogPermission() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); - conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); @@ -613,7 +610,6 @@ public void testUpdateTopicPropertiesAuthorization() throws Exception { public void testSubscriptionPrefixAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); - conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(TestAuthorizationProviderWithSubscriptionPrefix.class.getName()); setup(); @@ -698,7 +694,6 @@ public void testAuthData() throws Exception { public void testPermissionForProducerCreateInitialSubscription() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); - conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java index 81d65b192049b..2fc8aebf64a4a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java @@ -195,7 +195,7 @@ protected void setup() throws Exception { Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); conf.setSuperUserRoles(superUserRoles); - conf.setTopicLevelPoliciesEnabled(false); + conf.setAuthorizationEnabled(true); conf.setAuthenticationEnabled(true); Set providersClassNames = Sets.newHashSet(MutualAuthenticationProvider.class.getName()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java index 4d5e7deaf7d99..87f12e6acdcb2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java @@ -37,7 +37,6 @@ import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.slf4j.Logger; @@ -93,8 +92,6 @@ protected void setup() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); - conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); - conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); conf.setClusterName("test"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java index ba43ee6d6a2dd..fdf41c4a6ada1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java @@ -27,9 +27,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -41,7 +39,6 @@ import org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.ObjectMapperFactory; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,12 +87,11 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setBrokerClientAuthenticationPlugin(AuthenticationOAuth2.class.getName()); - final Map oauth2Param = new HashMap<>(); - oauth2Param.put("privateKey", CREDENTIALS_FILE); - oauth2Param.put("issuerUrl", server.getIssuer()); - oauth2Param.put("audience", audience); - conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory - .getMapper().getObjectMapper().writeValueAsString(oauth2Param)); + conf.setBrokerClientAuthenticationParameters("{\n" + + " \"privateKey\": \"" + CREDENTIALS_FILE + "\",\n" + + " \"issuerUrl\": \"" + server.getIssuer() + "\",\n" + + " \"audience\": \"" + audience + "\",\n" + + "}\n"); conf.setClusterName("test"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java index 77405e142013a..8e508b6cf2068 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java @@ -32,7 +32,6 @@ import io.jsonwebtoken.SignatureAlgorithm; import lombok.Cleanup; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; @@ -50,7 +49,6 @@ import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -85,7 +83,6 @@ protected void cleanup() throws Exception { super.internalCleanup(); } - @SneakyThrows protected void internalSetUpForBroker() { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); @@ -117,25 +114,6 @@ protected void internalSetUpForBroker() { conf.setAuthenticationProviders(providers); conf.setNumExecutorThreadPoolSize(5); - Set tlsProtocols = Sets.newConcurrentHashSet(); - tlsProtocols.add("TLSv1.3"); - tlsProtocols.add("TLSv1.2"); - conf.setBrokerClientAuthenticationPlugin(AuthenticationKeyStoreTls.class.getName()); - Map authParams = new HashMap<>(); - authParams.put(AuthenticationKeyStoreTls.KEYSTORE_TYPE, KEYSTORE_TYPE); - authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH); - authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW); - conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory.getMapper() - .getObjectMapper().writeValueAsString(authParams)); - conf.setBrokerClientTlsEnabled(true); - conf.setBrokerClientTlsEnabledWithKeyStore(true); - conf.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); - conf.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); - conf.setBrokerClientTlsKeyStore(CLIENT_KEYSTORE_FILE_PATH); - conf.setBrokerClientTlsKeyStoreType(KEYSTORE_TYPE); - conf.setBrokerClientTlsKeyStorePassword(CLIENT_KEYSTORE_PW); - conf.setBrokerClientTlsProtocols(tlsProtocols); - } protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java index b9139dabdf021..76936334eb0ba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java @@ -85,7 +85,6 @@ public void setup() throws Exception { // set isTcpLookup = true, to use BinaryProtoLookupService to get topics for a pattern. isTcpLookup = true; - conf.setTopicLevelPoliciesEnabled(false); conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); diff --git a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf index 4e2fd40298354..d9411e655ad5b 100644 --- a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf +++ b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf @@ -29,5 +29,4 @@ authenticationEnabled=true authenticationProviders=org.apache.pulsar.MockTokenAuthenticationProvider brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= -loadBalancerOverrideBrokerNicSpeedGbps=2 -topicLevelPoliciesEnabled=false \ No newline at end of file +loadBalancerOverrideBrokerNicSpeedGbps=2 \ No newline at end of file diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java index 9c8e5197adf1a..8229d929ee5e3 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java @@ -168,7 +168,7 @@ protected void setup() throws Exception { conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); // Expires after an hour conf.setBrokerClientAuthenticationParameters( - "entityType:admin,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000)); + "entityType:broker,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000)); Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java index b7cfb87474707..99af3b1cf6abe 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java @@ -53,7 +53,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("authParam:admin"); + conf.setBrokerClientAuthenticationParameters("authParam:broker"); conf.setAuthenticateOriginalAuthData(true); Set superUserRoles = new HashSet(); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java index 3259cfd95c741..2c8c382b6a5ef 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java @@ -144,7 +144,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("authParam:admin"); + conf.setBrokerClientAuthenticationParameters("authParam:broker"); Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java index 2d97a4b06a856..e8bb128c8c190 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java @@ -78,8 +78,6 @@ public class ProxyWithAuthorizationNegTest extends ProducerConsumerBase { protected void setup() throws Exception { // enable tls and auth&auth at broker - conf.setTopicLevelPoliciesEnabled(false); - conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java index 7b550b7270f37..9119ffed4e28f 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java @@ -38,7 +38,6 @@ import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.AuthenticationFactory; -import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -64,9 +63,6 @@ public void setup() throws Exception { conf.setProperties(properties); conf.setSuperUserRoles(Sets.newHashSet(SUPER_USER_ROLE)); conf.setClusterName("c1"); - conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); - conf.setBrokerClientAuthenticationParameters("token:" + AuthTokenUtils - .createToken(secretKey, SUPER_USER_ROLE, Optional.empty())); internalSetup(); admin.clusters().createCluster("c1", ClusterData.builder().build()); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 49e5ae378342d..057039edc3be2 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -89,7 +89,6 @@ public void setup() throws Exception { "org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder"); brokerEnvs.put("forceDeleteNamespaceAllowed", "true"); brokerEnvs.put("loadBalancerDebugModeEnabled", "true"); - brokerEnvs.put("topicLevelPoliciesEnabled", "false"); brokerEnvs.put("PULSAR_MEM", "-Xmx512M"); spec.brokerEnvs(brokerEnvs); pulsarCluster = PulsarCluster.forSpec(spec); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java index 87db46f2bb625..0a9bb5e19592a 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java @@ -68,7 +68,6 @@ protected void beforeStartCluster() { envMap.put("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(secretKey)); envMap.put("superUserRoles", "admin"); envMap.put("brokerDeleteInactiveTopicsEnabled", "false"); - envMap.put("topicLevelPoliciesEnabled", "false"); for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { brokerContainer.withEnv(envMap); From 5dc11a5d4edbcbc3a13d4e20c9b510ed99b007c9 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Sun, 8 Oct 2023 16:11:33 +0800 Subject: [PATCH 308/494] [fix] [metadata] Fix zookeeper related flacky test (#21310) Fix zookeeper related flacky test (cherry picked from commit 9ab7417edc20b6618bac8f66921815c7b9d5e5b8) --- pulsar-metadata/pom.xml | 7 + .../replication/AuditorBookieTest.java | 1 - .../AuditorCheckAllLedgersTaskTest.java | 1 - .../replication/AuditorLedgerCheckerTest.java | 1 - .../AuditorPeriodicBookieCheckTest.java | 1 - .../replication/AuditorPeriodicCheckTest.java | 1 - .../AuditorPlacementPolicyCheckTaskTest.java | 1 - .../AuditorPlacementPolicyCheckTest.java | 1 - .../AuditorReplicasCheckTaskTest.java | 1 - .../replication/AuditorReplicasCheckTest.java | 1 - .../AuditorRollingRestartTest.java | 1 - .../replication/AuthAutoRecoveryTest.java | 1 - .../replication/AutoRecoveryMainTest.java | 6 - .../BookKeeperClusterTestCase.java | 851 ++++++++++++++++++ .../replication/BookieAutoRecoveryTest.java | 1 - .../replication/BookieLedgerIndexTest.java | 1 - ...estAutoRecoveryAlongWithBookieServers.java | 1 - .../replication/TestReplicationWorker.java | 1 - .../bookkeeper/replication/ZooKeeperUtil.java | 215 +++++ .../test/BookKeeperClusterTestCase.java | 3 +- .../client/BookKeeperTestClient.java | 1 - 21 files changed, 1074 insertions(+), 24 deletions(-) create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java create mode 100644 pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ZooKeeperUtil.java diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 421d461d3b0bf..3bf76b8336783 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -86,6 +86,13 @@ + + ${project.groupId} + testmocks + ${project.version} + test + + org.xerial.snappy snappy-java diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java index 9750fb52d41a3..14cdf3e1fc29c 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorBookieTest.java @@ -31,7 +31,6 @@ import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.proto.BookieServer; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerAuditorManager; import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java index 5c0a3f39325e5..6b58c72af0766 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java @@ -30,7 +30,6 @@ import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; import org.apache.bookkeeper.stats.NullStatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index ffd71f9311f42..ec5f77f79464b 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -67,7 +67,6 @@ import org.apache.bookkeeper.proto.BookieServer; import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; import org.apache.bookkeeper.stats.NullStatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java index 4acb207570a2d..9e8c5a54a5d91 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java @@ -30,7 +30,6 @@ import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; import org.apache.bookkeeper.net.BookieSocketAddress; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java index e3b0aa37d7e55..0fa6538b3ed43 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java @@ -66,7 +66,6 @@ import org.apache.bookkeeper.stats.Counter; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java index 1bafb8589d91a..8b9c0b143028a 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java @@ -31,7 +31,6 @@ import org.apache.bookkeeper.meta.LedgerManagerFactory; import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; import org.apache.bookkeeper.stats.NullStatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java index 159a4e88a33bd..5637819a9275b 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTest.java @@ -53,7 +53,6 @@ import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; import org.apache.bookkeeper.stats.Gauge; import org.apache.bookkeeper.stats.NullStatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java index 21dd2807b75d3..62162bd25f427 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java @@ -31,7 +31,6 @@ import org.apache.bookkeeper.meta.LedgerManagerFactory; import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; import org.apache.bookkeeper.stats.NullStatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java index a4d6d86deced2..2e9dbc158597d 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTest.java @@ -57,7 +57,6 @@ import org.apache.bookkeeper.stats.Gauge; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; import org.apache.bookkeeper.test.TestStatsProvider.TestOpStatsLogger; import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java index 2c458d635f528..3e5081ed0ef9d 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorRollingRestartTest.java @@ -29,7 +29,6 @@ import org.apache.bookkeeper.meta.LedgerManagerFactory; import org.apache.bookkeeper.meta.LedgerUnderreplicationManager; import org.apache.bookkeeper.net.BookieId; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestCallbacks; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java index db338d1bb4b39..41e159b77714f 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuthAutoRecoveryTest.java @@ -27,7 +27,6 @@ import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.proto.ClientConnectionPeer; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java index 62416968142b1..1d741c551ddb9 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AutoRecoveryMainTest.java @@ -25,10 +25,8 @@ import java.io.IOException; import java.lang.reflect.Field; import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.bookie.BookieImpl; import org.apache.bookkeeper.net.BookieId; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.util.TestUtils; import org.apache.pulsar.metadata.bookkeeper.PulsarLedgerManagerFactory; import org.apache.pulsar.metadata.bookkeeper.PulsarMetadataClientDriver; @@ -42,7 +40,6 @@ /** * Test the AuditorPeer. */ -@Slf4j public class AutoRecoveryMainTest extends BookKeeperClusterTestCase { public AutoRecoveryMainTest() throws Exception { @@ -68,7 +65,6 @@ public void tearDown() throws Exception { */ @Test public void testStartup() throws Exception { - log.info("testStartup()"); confByIndex(0).setMetadataServiceUri( zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); AutoRecoveryMain main = new AutoRecoveryMain(confByIndex(0)); @@ -89,7 +85,6 @@ public void testStartup() throws Exception { */ @Test public void testShutdown() throws Exception { - log.info("testShutdown()"); confByIndex(0).setMetadataServiceUri( zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); AutoRecoveryMain main = new AutoRecoveryMain(confByIndex(0)); @@ -113,7 +108,6 @@ public void testShutdown() throws Exception { */ @Test public void testAutoRecoverySessionLoss() throws Exception { - log.info("testAutoRecoverySessionLoss()"); confByIndex(0).setMetadataServiceUri( zkUtil.getMetadataServiceUri().replaceAll("zk://", "metadata-store:").replaceAll("/ledgers", "")); confByIndex(1).setMetadataServiceUri( diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java new file mode 100644 index 0000000000000..c681a1f0764ee --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java @@ -0,0 +1,851 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * This file is derived from BookKeeperClusterTestCase from Apache BookKeeper + * http://bookkeeper.apache.org + */ + +package org.apache.bookkeeper.replication; + +import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; +import static org.testng.Assert.assertFalse; +import com.google.common.base.Stopwatch; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.BookieException; +import org.apache.bookkeeper.client.BookKeeperTestClient; +import org.apache.bookkeeper.client.TestStatsProvider; +import org.apache.bookkeeper.common.allocator.PoolingPolicy; +import org.apache.bookkeeper.conf.AbstractConfiguration; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.conf.TestBKConfiguration; +import org.apache.bookkeeper.metastore.InMemoryMetaStore; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.test.ServerTester; +import org.apache.bookkeeper.test.TmpDirs; +import org.apache.bookkeeper.test.ZooKeeperCluster; +import org.apache.bookkeeper.test.ZooKeeperClusterUtil; +import org.apache.pulsar.common.util.PortManager; +import org.apache.pulsar.metadata.api.MetadataStoreConfig; +import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.Awaitility; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeTest; + +/** + * A class runs several bookie servers for testing. + */ +public abstract class BookKeeperClusterTestCase { + + static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterTestCase.class); + + protected String testName; + + @BeforeMethod + public void handleTestMethodName(Method method) { + testName = method.getName(); + } + + // Metadata service related variables + protected final ZooKeeperCluster zkUtil; + protected ZooKeeper zkc; + protected String metadataServiceUri; + protected FaultInjectionMetadataStore metadataStore; + + // BookKeeper related variables + protected final TmpDirs tmpDirs = new TmpDirs(); + protected final List servers = new LinkedList<>(); + + protected int numBookies; + protected BookKeeperTestClient bkc; + protected boolean useUUIDasBookieId = true; + + /* + * Loopback interface is set as the listening interface and allowloopback is + * set to true in this server config. So bookies in this test process would + * bind to loopback address. + */ + protected final ServerConfiguration baseConf = TestBKConfiguration.newServerConfiguration(); + protected final ClientConfiguration baseClientConf = TestBKConfiguration.newClientConfiguration(); + + private boolean isAutoRecoveryEnabled; + protected ExecutorService executor; + private final List bookiePorts = new ArrayList<>(); + + SynchronousQueue asyncExceptions = new SynchronousQueue<>(); + protected void captureThrowable(Runnable c) { + try { + c.run(); + } catch (Throwable e) { + LOG.error("Captured error: ", e); + asyncExceptions.add(e); + } + } + + public BookKeeperClusterTestCase(int numBookies) { + this(numBookies, 120); + } + + public BookKeeperClusterTestCase(int numBookies, int testTimeoutSecs) { + this(numBookies, 1, testTimeoutSecs); + } + + public BookKeeperClusterTestCase(int numBookies, int numOfZKNodes, int testTimeoutSecs) { + this.numBookies = numBookies; + if (numOfZKNodes == 1) { + zkUtil = new ZooKeeperUtil(getLedgersRootPath()); + } else { + try { + zkUtil = new ZooKeeperClusterUtil(numOfZKNodes); + } catch (IOException | KeeperException | InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + @BeforeTest + public void setUp() throws Exception { + setUp(getLedgersRootPath()); + } + + protected void setUp(String ledgersRootPath) throws Exception { + LOG.info("Setting up test {}", getClass()); + InMemoryMetaStore.reset(); + setMetastoreImplClass(baseConf); + setMetastoreImplClass(baseClientConf); + executor = Executors.newCachedThreadPool(); + + Stopwatch sw = Stopwatch.createStarted(); + try { + // start zookeeper service + startZKCluster(); + // start bookkeeper service + this.metadataServiceUri = getMetadataServiceUri(ledgersRootPath); + startBKCluster(metadataServiceUri); + LOG.info("Setup testcase {} @ metadata service {} in {} ms.", + testName, metadataServiceUri, sw.elapsed(TimeUnit.MILLISECONDS)); + } catch (Exception e) { + LOG.error("Error setting up", e); + throw e; + } + } + + protected String getMetadataServiceUri(String ledgersRootPath) { + return zkUtil.getMetadataServiceUri(ledgersRootPath); + } + + private String getLedgersRootPath() { + return changeLedgerPath() + "/ledgers"; + } + + protected String changeLedgerPath() { + return ""; + } + + @AfterTest(alwaysRun = true) + public void tearDown() throws Exception { + boolean failed = false; + for (Throwable e : asyncExceptions) { + LOG.error("Got async exception: ", e); + failed = true; + } + assertFalse(failed, "Async failure"); + Stopwatch sw = Stopwatch.createStarted(); + LOG.info("TearDown"); + Exception tearDownException = null; + // stop bookkeeper service + try { + stopBKCluster(); + } catch (Exception e) { + LOG.error("Got Exception while trying to stop BKCluster", e); + tearDownException = e; + } + // stop zookeeper service + try { + // cleanup for metrics. + metadataStore.close(); + stopZKCluster(); + } catch (Exception e) { + LOG.error("Got Exception while trying to stop ZKCluster", e); + tearDownException = e; + } + // cleanup temp dirs + try { + tmpDirs.cleanup(); + } catch (Exception e) { + LOG.error("Got Exception while trying to cleanupTempDirs", e); + tearDownException = e; + } + + executor.shutdownNow(); + + LOG.info("Tearing down test {} in {} ms.", testName, sw.elapsed(TimeUnit.MILLISECONDS)); + if (tearDownException != null) { + throw tearDownException; + } + } + + /** + * Start zookeeper cluster. + * + * @throws Exception + */ + protected void startZKCluster() throws Exception { + zkUtil.startCluster(); + zkc = zkUtil.getZooKeeperClient(); + metadataStore = new FaultInjectionMetadataStore( + MetadataStoreExtended.create(zkUtil.getZooKeeperConnectString(), + MetadataStoreConfig.builder().build())); + } + + /** + * Stop zookeeper cluster. + * + * @throws Exception + */ + protected void stopZKCluster() throws Exception { + zkUtil.killCluster(); + } + + /** + * Start cluster. Also, starts the auto recovery process for each bookie, if + * isAutoRecoveryEnabled is true. + * + * @throws Exception + */ + protected void startBKCluster(String metadataServiceUri) throws Exception { + baseConf.setMetadataServiceUri(metadataServiceUri); + baseClientConf.setMetadataServiceUri(metadataServiceUri); + baseClientConf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap); + + if (numBookies > 0) { + bkc = new BookKeeperTestClient(baseClientConf, new TestStatsProvider()); + } + + // Create Bookie Servers (B1, B2, B3) + for (int i = 0; i < numBookies; i++) { + bookiePorts.add(startNewBookie()); + } + } + + /** + * Stop cluster. Also, stops all the auto recovery processes for the bookie + * cluster, if isAutoRecoveryEnabled is true. + * + * @throws Exception + */ + protected void stopBKCluster() throws Exception { + if (bkc != null) { + bkc.close(); + } + + for (ServerTester t : servers) { + t.shutdown(); + } + servers.clear(); + bookiePorts.removeIf(PortManager::releaseLockedPort); + } + + protected ServerConfiguration newServerConfiguration() throws Exception { + File f = tmpDirs.createNew("bookie", "test"); + + int port; + if (baseConf.isEnableLocalTransport() || !baseConf.getAllowEphemeralPorts()) { + port = nextLockedFreePort(); + } else { + port = 0; + } + return newServerConfiguration(port, f, new File[] { f }); + } + + protected ClientConfiguration newClientConfiguration() { + return new ClientConfiguration(baseConf); + } + + protected ServerConfiguration newServerConfiguration(int port, File journalDir, File[] ledgerDirs) { + ServerConfiguration conf = new ServerConfiguration(baseConf); + conf.setBookiePort(port); + conf.setJournalDirName(journalDir.getPath()); + String[] ledgerDirNames = new String[ledgerDirs.length]; + for (int i = 0; i < ledgerDirs.length; i++) { + ledgerDirNames[i] = ledgerDirs[i].getPath(); + } + conf.setLedgerDirNames(ledgerDirNames); + conf.setEnableTaskExecutionStats(true); + conf.setAllocatorPoolingPolicy(PoolingPolicy.UnpooledHeap); + return conf; + } + + protected void stopAllBookies() throws Exception { + stopAllBookies(true); + } + + protected void stopAllBookies(boolean shutdownClient) throws Exception { + for (ServerTester t : servers) { + t.shutdown(); + } + servers.clear(); + if (shutdownClient && bkc != null) { + bkc.close(); + bkc = null; + } + } + + protected String newMetadataServiceUri(String ledgersRootPath) { + return zkUtil.getMetadataServiceUri(ledgersRootPath); + } + + protected String newMetadataServiceUri(String ledgersRootPath, String type) { + return zkUtil.getMetadataServiceUri(ledgersRootPath, type); + } + + /** + * Get bookie address for bookie at index. + */ + public BookieId getBookie(int index) throws Exception { + return servers.get(index).getServer().getBookieId(); + } + + protected List bookieAddresses() throws Exception { + List bookieIds = new ArrayList<>(); + for (ServerTester a : servers) { + bookieIds.add(a.getServer().getBookieId()); + } + return bookieIds; + } + + protected List bookieLedgerDirs() throws Exception { + return servers.stream() + .flatMap(t -> Arrays.stream(t.getConfiguration().getLedgerDirs())) + .collect(Collectors.toList()); + } + + protected List bookieJournalDirs() throws Exception { + return servers.stream() + .flatMap(t -> Arrays.stream(t.getConfiguration().getJournalDirs())) + .collect(Collectors.toList()); + } + + protected BookieId addressByIndex(int index) throws Exception { + return servers.get(index).getServer().getBookieId(); + } + + protected BookieServer serverByIndex(int index) throws Exception { + return servers.get(index).getServer(); + } + + protected ServerConfiguration confByIndex(int index) throws Exception { + return servers.get(index).getConfiguration(); + } + + private Optional byAddress(BookieId addr) throws UnknownHostException { + for (ServerTester s : servers) { + if (s.getServer().getBookieId().equals(addr)) { + return Optional.of(s); + } + } + return Optional.empty(); + } + + protected int indexOfServer(BookieServer b) throws Exception { + for (int i = 0; i < servers.size(); i++) { + if (servers.get(i).getServer().equals(b)) { + return i; + } + } + return -1; + } + + protected int lastBookieIndex() { + return servers.size() - 1; + } + + protected int bookieCount() { + return servers.size(); + } + + private OptionalInt indexByAddress(BookieId addr) throws UnknownHostException { + for (int i = 0; i < servers.size(); i++) { + if (addr.equals(servers.get(i).getServer().getBookieId())) { + return OptionalInt.of(i); + } + } + return OptionalInt.empty(); + } + + /** + * Get bookie configuration for bookie. + */ + public ServerConfiguration getBkConf(BookieId addr) throws Exception { + return byAddress(addr).get().getConfiguration(); + } + + /** + * Kill a bookie by its socket address. Also, stops the autorecovery process + * for the corresponding bookie server, if isAutoRecoveryEnabled is true. + * + * @param addr + * Socket Address + * @return the configuration of killed bookie + * @throws InterruptedException + */ + public ServerConfiguration killBookie(BookieId addr) throws Exception { + Optional tester = byAddress(addr); + if (tester.isPresent()) { + if (tester.get().autoRecovery != null + && tester.get().autoRecovery.getAuditor() != null + && tester.get().autoRecovery.getAuditor().isRunning()) { + LOG.warn("Killing bookie {} who is the current Auditor", addr); + } + servers.remove(tester.get()); + tester.get().shutdown(); + return tester.get().getConfiguration(); + } + return null; + } + + /** + * Set the bookie identified by its socket address to readonly. + * + * @param addr + * Socket Address + * @throws InterruptedException + */ + public void setBookieToReadOnly(BookieId addr) throws Exception { + Optional tester = byAddress(addr); + if (tester.isPresent()) { + tester.get().getServer().getBookie().getStateManager().transitionToReadOnlyMode().get(); + } + } + + /** + * Kill a bookie by index. Also, stops the respective auto recovery process + * for this bookie, if isAutoRecoveryEnabled is true. + * + * @param index + * Bookie Index + * @return the configuration of killed bookie + * @throws InterruptedException + * @throws IOException + */ + public ServerConfiguration killBookie(int index) throws Exception { + ServerTester tester = servers.remove(index); + tester.shutdown(); + return tester.getConfiguration(); + } + + /** + * Kill bookie by index and verify that it's stopped. + * + * @param index index of bookie to kill + * + * @return configuration of killed bookie + */ + public ServerConfiguration killBookieAndWaitForZK(int index) throws Exception { + ServerTester tester = servers.get(index); // IKTODO: this method is awful + ServerConfiguration ret = killBookie(index); + while (zkc.exists("/ledgers/" + AVAILABLE_NODE + "/" + + tester.getServer().getBookieId().toString(), false) != null) { + Thread.sleep(500); + } + return ret; + } + + /** + * Sleep a bookie. + * + * @param addr + * Socket Address + * @param seconds + * Sleep seconds + * @return Count Down latch which will be counted down just after sleep begins + * @throws InterruptedException + * @throws IOException + */ + public CountDownLatch sleepBookie(BookieId addr, final int seconds) + throws Exception { + Optional tester = byAddress(addr); + if (tester.isPresent()) { + CountDownLatch latch = new CountDownLatch(1); + Thread sleeper = new Thread() { + @Override + public void run() { + try { + tester.get().getServer().suspendProcessing(); + LOG.info("bookie {} is asleep", tester.get().getAddress()); + latch.countDown(); + Thread.sleep(seconds * 1000); + tester.get().getServer().resumeProcessing(); + LOG.info("bookie {} is awake", tester.get().getAddress()); + } catch (Exception e) { + LOG.error("Error suspending bookie", e); + } + } + }; + sleeper.start(); + return latch; + } else { + throw new IOException("Bookie not found"); + } + } + + /** + * Sleep a bookie until I count down the latch. + * + * @param addr + * Socket Address + * @param l + * Latch to wait on + * @throws InterruptedException + * @throws IOException + */ + public void sleepBookie(BookieId addr, final CountDownLatch l) + throws InterruptedException, IOException { + final CountDownLatch suspendLatch = new CountDownLatch(1); + sleepBookie(addr, l, suspendLatch); + suspendLatch.await(); + } + + public void sleepBookie(BookieId addr, final CountDownLatch l, final CountDownLatch suspendLatch) + throws InterruptedException, IOException { + Optional tester = byAddress(addr); + if (tester.isPresent()) { + BookieServer bookie = tester.get().getServer(); + LOG.info("Sleep bookie {}.", addr); + Thread sleeper = new Thread() { + @Override + public void run() { + try { + bookie.suspendProcessing(); + if (null != suspendLatch) { + suspendLatch.countDown(); + } + l.await(); + bookie.resumeProcessing(); + } catch (Exception e) { + LOG.error("Error suspending bookie", e); + } + } + }; + sleeper.start(); + } else { + throw new IOException("Bookie not found"); + } + } + + /** + * Restart bookie servers. Also restarts all the respective auto recovery + * process, if isAutoRecoveryEnabled is true. + * + * @throws InterruptedException + * @throws IOException + * @throws KeeperException + * @throws BookieException + */ + public void restartBookies() + throws Exception { + restartBookies(c -> c); + } + + /** + * Restart a bookie. Also restart the respective auto recovery process, + * if isAutoRecoveryEnabled is true. + * + * @param addr + * @throws InterruptedException + * @throws IOException + * @throws KeeperException + * @throws BookieException + */ + public void restartBookie(BookieId addr) throws Exception { + OptionalInt toRemove = indexByAddress(addr); + if (toRemove.isPresent()) { + ServerConfiguration newConfig = killBookie(toRemove.getAsInt()); + Thread.sleep(1000); + startAndAddBookie(newConfig); + } else { + throw new IOException("Bookie not found"); + } + } + + public void restartBookies(Function reconfFunction) + throws Exception { + // shut down bookie server + List confs = new ArrayList<>(); + for (ServerTester server : servers) { + server.shutdown(); + confs.add(server.getConfiguration()); + } + servers.clear(); + Thread.sleep(1000); + // restart them to ensure we can't + for (ServerConfiguration conf : confs) { + // ensure the bookie port is loaded correctly + startAndAddBookie(reconfFunction.apply(conf)); + } + } + + /** + * Helper method to startup a new bookie server with the indicated port + * number. Also, starts the auto recovery process, if the + * isAutoRecoveryEnabled is set true. + * + * @throws IOException + */ + public int startNewBookie() + throws Exception { + return startNewBookieAndReturnAddress().getPort(); + } + + public BookieSocketAddress startNewBookieAndReturnAddress() + throws Exception { + ServerConfiguration conf = newServerConfiguration(); + LOG.info("Starting new bookie on port: {}", conf.getBookiePort()); + return startAndAddBookie(conf).getServer().getLocalAddress(); + } + + public BookieId startNewBookieAndReturnBookieId() + throws Exception { + ServerConfiguration conf = newServerConfiguration(); + LOG.info("Starting new bookie on port: {}", conf.getBookiePort()); + return startAndAddBookie(conf).getServer().getBookieId(); + } + + protected ServerTester startAndAddBookie(ServerConfiguration conf) throws Exception { + ServerTester server = startBookie(conf); + servers.add(server); + return server; + } + + protected ServerTester startAndAddBookie(ServerConfiguration conf, Bookie b) throws Exception { + ServerTester server = startBookie(conf, b); + servers.add(server); + return server; + } + /** + * Helper method to startup a bookie server using a configuration object. + * Also, starts the auto recovery process if isAutoRecoveryEnabled is true. + * + * @param conf + * Server Configuration Object + * + */ + protected ServerTester startBookie(ServerConfiguration conf) + throws Exception { + ServerTester tester = new ServerTester(conf); + + if (bkc == null) { + bkc = new BookKeeperTestClient(baseClientConf, new TestStatsProvider()); + } + + BookieId address = tester.getServer().getBookieId(); + Future waitForBookie = conf.isForceReadOnlyBookie() + ? bkc.waitForReadOnlyBookie(address) + : bkc.waitForWritableBookie(address); + + tester.getServer().start(); + + waitForBookie.get(30, TimeUnit.SECONDS); + LOG.info("New bookie '{}' has been created.", address); + + if (isAutoRecoveryEnabled()) { + tester.startAutoRecovery(); + } + + int port = conf.getBookiePort(); + + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> { + while (zkc.exists("/ledgers/" + AVAILABLE_NODE + "/" + + tester.getServer().getBookieId().toString(), false) == null) { + Thread.sleep(100); + } + return true; + }); + bkc.readBookiesBlocking(); + + LOG.info("New bookie on port " + port + " has been created."); + + return tester; + } + + /** + * Start a bookie with the given bookie instance. Also, starts the auto + * recovery for this bookie, if isAutoRecoveryEnabled is true. + */ + protected ServerTester startBookie(ServerConfiguration conf, final Bookie b) + throws Exception { + ServerTester tester = new ServerTester(conf, b); + if (bkc == null) { + bkc = new BookKeeperTestClient(baseClientConf, new TestStatsProvider()); + } + BookieId address = tester.getServer().getBookieId(); + Future waitForBookie = conf.isForceReadOnlyBookie() + ? bkc.waitForReadOnlyBookie(address) + : bkc.waitForWritableBookie(address); + + tester.getServer().start(); + + waitForBookie.get(30, TimeUnit.SECONDS); + + if (isAutoRecoveryEnabled()) { + tester.startAutoRecovery(); + } + + int port = conf.getBookiePort(); + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> + metadataStore.exists( + getLedgersRootPath() + "/available/" + address).join() + ); + bkc.readBookiesBlocking(); + + LOG.info("New bookie '{}' has been created.", address); + return tester; + } + + public void setMetastoreImplClass(AbstractConfiguration conf) { + conf.setMetastoreImplClass(InMemoryMetaStore.class.getName()); + } + + /** + * Flags used to enable/disable the auto recovery process. If it is enabled, + * starting the bookie server will starts the auto recovery process for that + * bookie. Also, stopping bookie will stops the respective auto recovery + * process. + * + * @param isAutoRecoveryEnabled + * Value true will enable the auto recovery process. Value false + * will disable the auto recovery process + */ + public void setAutoRecoveryEnabled(boolean isAutoRecoveryEnabled) { + this.isAutoRecoveryEnabled = isAutoRecoveryEnabled; + } + + /** + * Flag used to check whether auto recovery process is enabled/disabled. By + * default the flag is false. + * + * @return true, if the auto recovery is enabled. Otherwise return false. + */ + public boolean isAutoRecoveryEnabled() { + return isAutoRecoveryEnabled; + } + + /** + * Will starts the auto recovery process for the bookie servers. One auto + * recovery process per each bookie server, if isAutoRecoveryEnabled is + * enabled. + */ + public void startReplicationService() throws Exception { + for (ServerTester t : servers) { + t.startAutoRecovery(); + } + } + + /** + * Will stops all the auto recovery processes for the bookie cluster, if + * isAutoRecoveryEnabled is true. + */ + public void stopReplicationService() throws Exception{ + for (ServerTester t : servers) { + t.stopAutoRecovery(); + } + } + + public Auditor getAuditor(int timeout, TimeUnit unit) throws Exception { + final long timeoutAt = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, unit); + while (System.nanoTime() < timeoutAt) { + for (ServerTester t : servers) { + Auditor a = t.getAuditor(); + ReplicationWorker replicationWorker = t.getReplicationWorker(); + + // found a candidate Auditor + ReplicationWorker + if (a != null && a.isRunning() + && replicationWorker != null && replicationWorker.isRunning()) { + int deathWatchInterval = t.getConfiguration().getDeathWatchInterval(); + Thread.sleep(deathWatchInterval + 1000); + } + + // double check, because in the meantime AutoRecoveryDeathWatcher may have killed the + // AutoRecovery daemon + if (a != null && a.isRunning() + && replicationWorker != null && replicationWorker.isRunning()) { + LOG.info("Found Auditor Bookie {}", t.getServer().getBookieId()); + return a; + } + } + Thread.sleep(100); + } + throw new Exception("No auditor found"); + } + + /** + * Check whether the InetSocketAddress was created using a hostname or an IP + * address. Represent as 'hostname/IPaddress' if the InetSocketAddress was + * created using hostname. Represent as '/IPaddress' if the + * InetSocketAddress was created using an IPaddress + * + * @param bookieId id + * @return true if the address was created using an IP address, false if the + * address was created using a hostname + */ + public boolean isCreatedFromIp(BookieId bookieId) { + BookieSocketAddress addr = bkc.getBookieAddressResolver().resolve(bookieId); + return addr.getSocketAddress().toString().startsWith("/"); + } + + public void resetBookieOpLoggers() { + servers.forEach(t -> t.getStatsProvider().clear()); + } + + public TestStatsProvider getStatsProvider(BookieId addr) throws UnknownHostException { + return byAddress(addr).get().getStatsProvider(); + } + + public TestStatsProvider getStatsProvider(int index) throws Exception { + return servers.get(index).getStatsProvider(); + } + +} diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java index c8c76302b89e1..888303d3e665c 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieAutoRecoveryTest.java @@ -44,7 +44,6 @@ import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.proto.BookieServer; import org.apache.bookkeeper.stats.NullStatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java index eb9f95ffdf7a5..1d5cf868cce65 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java @@ -35,7 +35,6 @@ import org.apache.bookkeeper.meta.LayoutManager; import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerManagerFactory; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.bookkeeper.PulsarLayoutManager; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java index 8a2e7f2747a22..11797c8373715 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestAutoRecoveryAlongWithBookieServers.java @@ -27,7 +27,6 @@ import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.net.BookieId; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.util.BookKeeperConstants; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java index ca02f91d1de36..7938feaba19fe 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/TestReplicationWorker.java @@ -78,7 +78,6 @@ import org.apache.bookkeeper.stats.Gauge; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.stats.StatsLogger; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.test.TestStatsProvider; import org.apache.bookkeeper.test.TestStatsProvider.TestStatsLogger; import org.apache.bookkeeper.util.BookKeeperConstants; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ZooKeeperUtil.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ZooKeeperUtil.java new file mode 100644 index 0000000000000..5113edb72c49a --- /dev/null +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/ZooKeeperUtil.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * This file is derived from ZooKeeperUtil from Apache BookKeeper + * http://bookkeeper.apache.org + */ + +package org.apache.bookkeeper.replication; + +import static org.testng.Assert.assertTrue; +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.apache.bookkeeper.test.ZooKeeperCluster; +import org.apache.bookkeeper.util.IOUtils; +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.bookkeeper.zookeeper.ZooKeeperClient; +import org.apache.commons.io.FileUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.test.ClientBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test the zookeeper utilities. + */ +public class ZooKeeperUtil implements ZooKeeperCluster { + + static { + // org.apache.zookeeper.test.ClientBase uses FourLetterWordMain, from 3.5.3 four letter words + // are disabled by default due to security reasons + System.setProperty("zookeeper.4lw.commands.whitelist", "*"); + } + static final Logger LOG = LoggerFactory.getLogger(ZooKeeperUtil.class); + + // ZooKeeper related variables + protected Integer zooKeeperPort = 0; + private InetSocketAddress zkaddr; + + protected ZooKeeperServer zks; + protected ZooKeeper zkc; // zookeeper client + protected NIOServerCnxnFactory serverFactory; + protected File zkTmpDir; + private String connectString; + private String ledgersRootPath; + + public ZooKeeperUtil(String ledgersRootPath) { + this.ledgersRootPath = ledgersRootPath; + String loopbackIPAddr = InetAddress.getLoopbackAddress().getHostAddress(); + zkaddr = new InetSocketAddress(loopbackIPAddr, 0); + connectString = loopbackIPAddr + ":" + zooKeeperPort; + } + + @Override + public ZooKeeper getZooKeeperClient() { + return zkc; + } + + @Override + public String getZooKeeperConnectString() { + return connectString; + } + + @Override + public String getMetadataServiceUri() { + return getMetadataServiceUri("/ledgers"); + } + + @Override + public String getMetadataServiceUri(String zkLedgersRootPath) { + return "zk://" + connectString + zkLedgersRootPath; + } + + @Override + public String getMetadataServiceUri(String zkLedgersRootPath, String type) { + return "zk+" + type + "://" + connectString + zkLedgersRootPath; + } + + @Override + public void startCluster() throws Exception { + // create a ZooKeeper server(dataDir, dataLogDir, port) + LOG.debug("Running ZK server"); + ClientBase.setupTestEnv(); + zkTmpDir = IOUtils.createTempDir("zookeeper", "test"); + + // start the server and client. + restartCluster(); + + // create default bk ensemble + createBKEnsemble(ledgersRootPath); + } + + @Override + public void createBKEnsemble(String ledgersPath) throws KeeperException, InterruptedException { + int last = ledgersPath.lastIndexOf('/'); + if (last > 0) { + String pathToCreate = ledgersPath.substring(0, last); + CompletableFuture future = new CompletableFuture<>(); + if (zkc.exists(pathToCreate, false) == null) { + ZkUtils.asyncCreateFullPathOptimistic(zkc, + pathToCreate, + new byte[0], + ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, (i, s, o, s1) -> { + future.complete(null); + }, null); + } + future.join(); + } + + ZooKeeperCluster.super.createBKEnsemble(ledgersPath); + } + @Override + public void restartCluster() throws Exception { + zks = new ZooKeeperServer(zkTmpDir, zkTmpDir, + ZooKeeperServer.DEFAULT_TICK_TIME); + serverFactory = new NIOServerCnxnFactory(); + serverFactory.configure(zkaddr, 100); + serverFactory.startup(zks); + + if (0 == zooKeeperPort) { + zooKeeperPort = serverFactory.getLocalPort(); + zkaddr = new InetSocketAddress(zkaddr.getAddress().getHostAddress(), zooKeeperPort); + connectString = zkaddr.getAddress().getHostAddress() + ":" + zooKeeperPort; + } + + boolean b = ClientBase.waitForServerUp(getZooKeeperConnectString(), + ClientBase.CONNECTION_TIMEOUT); + LOG.debug("Server up: " + b); + + // create a zookeeper client + LOG.debug("Instantiate ZK Client"); + zkc = ZooKeeperClient.newBuilder() + .connectString(getZooKeeperConnectString()) + .sessionTimeoutMs(10000) + .build(); + } + + @Override + public void sleepCluster(final int time, + final TimeUnit timeUnit, + final CountDownLatch l) + throws InterruptedException, IOException { + Thread[] allthreads = new Thread[Thread.activeCount()]; + Thread.enumerate(allthreads); + for (final Thread t : allthreads) { + if (t.getName().contains("SyncThread:0")) { + Thread sleeper = new Thread() { + @SuppressWarnings("deprecation") + public void run() { + try { + t.suspend(); + l.countDown(); + timeUnit.sleep(time); + t.resume(); + } catch (Exception e) { + LOG.error("Error suspending thread", e); + } + } + }; + sleeper.start(); + return; + } + } + throw new IOException("ZooKeeper thread not found"); + } + + @Override + public void stopCluster() throws Exception { + if (zkc != null) { + zkc.close(); + } + + // shutdown ZK server + if (serverFactory != null) { + serverFactory.shutdown(); + assertTrue(ClientBase.waitForServerDown(getZooKeeperConnectString(), ClientBase.CONNECTION_TIMEOUT), + "waiting for server down"); + } + if (zks != null) { + zks.getTxnLogFactory().close(); + } + } + + @Override + public void killCluster() throws Exception { + stopCluster(); + FileUtils.deleteDirectory(zkTmpDir); + } +} diff --git a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java index 40c2041d4e6c4..43db5ad4ba845 100644 --- a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java +++ b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java @@ -70,7 +70,6 @@ import org.apache.bookkeeper.meta.LedgerManager; import org.apache.bookkeeper.meta.LedgerManagerFactory; import org.apache.bookkeeper.meta.MetadataBookieDriver; -import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; import org.apache.bookkeeper.metastore.InMemoryMetaStore; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; @@ -486,7 +485,7 @@ public ServerConfiguration killBookie(int index) throws Exception { public ServerConfiguration killBookieAndWaitForZK(int index) throws Exception { ServerTester tester = servers.get(index); // IKTODO: this method is awful ServerConfiguration ret = killBookie(index); - while (zkc.exists(ZKMetadataDriverBase.resolveZkLedgersRootPath(baseConf) + "/" + AVAILABLE_NODE + "/" + while (zkc.exists("/ledgers/" + AVAILABLE_NODE + "/" + tester.getServer().getBookieId().toString(), false) != null) { Thread.sleep(500); } diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java b/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java index d023427e3be31..dd33c2c4532bf 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/BookKeeperTestClient.java @@ -52,7 +52,6 @@ public BookKeeperTestClient(ClientConfiguration conf, ZooKeeper zkc) throws IOException, InterruptedException, BKException { super(conf, zkc, null, new UnpooledByteBufAllocator(false), NullStatsLogger.INSTANCE, null, null, null); - this.statsProvider = statsProvider; } public BookKeeperTestClient(ClientConfiguration conf) From f1f417cff4cb2bbd9ae1f318c9ed728b2cffd080 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 9 Oct 2023 15:51:16 +0300 Subject: [PATCH 309/494] [fix][test] Fix flaky test NarUnpackerTest (#21328) (cherry picked from commit e76a86e3cd1c362e9daa1c88eb8b888e6ab38ab4) # Conflicts: # pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java --- .../pulsar/common/nar/NarUnpackerTest.java | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java index c6c5ee180f69a..f93afac2ff93b 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/nar/NarUnpackerTest.java @@ -46,7 +46,7 @@ public class NarUnpackerTest { public void createSampleZipFile() throws IOException { sampleZipFile = Files.createTempFile("sample", ".zip").toFile(); try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(sampleZipFile))) { - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 5000; i++) { ZipEntry e = new ZipEntry("hello" + i + ".txt"); out.putNextEntry(e); byte[] msg = "hello world!".getBytes(StandardCharsets.UTF_8); @@ -58,12 +58,20 @@ public void createSampleZipFile() throws IOException { } @AfterMethod(alwaysRun = true) - void deleteSampleZipFile() throws IOException { - if (sampleZipFile != null) { - sampleZipFile.delete(); + void deleteSampleZipFile() { + if (sampleZipFile != null && sampleZipFile.exists()) { + try { + sampleZipFile.delete(); + } catch (Exception e) { + log.warn("Failed to delete file {}", sampleZipFile, e); + } } - if (extractDirectory != null) { - FileUtils.deleteFile(extractDirectory, true); + if (extractDirectory != null && extractDirectory.exists()) { + try { + FileUtils.deleteFile(extractDirectory, true); + } catch (IOException e) { + log.warn("Failed to delete directory {}", extractDirectory, e); + } } } @@ -111,7 +119,7 @@ public static void main(String[] args) { @Test void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { - int processes = 10; + int processes = 5; String javaExePath = findJavaExe().getAbsolutePath(); CountDownLatch countDownLatch = new CountDownLatch(processes); AtomicInteger exceptionCounter = new AtomicInteger(); @@ -122,7 +130,9 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { // fork a new process with the same classpath Process process = new ProcessBuilder() .command(javaExePath, - "-Xmx64m", + "-Xmx96m", + "-XX:TieredStopAtLevel=1", + "-Dlog4j2.disable.jmx=true", "-cp", System.getProperty("java.class.path"), // use NarUnpackerWorker as the main class @@ -130,6 +140,7 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { // pass arguments to use for testing sampleZipFile.getAbsolutePath(), extractDirectory.getAbsolutePath()) + .redirectErrorStream(true) .start(); String output = IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8); int retval = process.waitFor(); @@ -147,7 +158,7 @@ void shouldExtractFilesOnceInDifferentProcess() throws InterruptedException { } }).start(); } - assertTrue(countDownLatch.await(30, TimeUnit.SECONDS)); + assertTrue(countDownLatch.await(30, TimeUnit.SECONDS), "All processes should finish before timeout"); assertEquals(exceptionCounter.get(), 0); assertEquals(extractCounter.get(), 1); } From 9ad081ccf1faef02248979884d9f0f413110707b Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 10 Oct 2023 23:45:10 +0800 Subject: [PATCH 310/494] [fix][sec] Fix MultiRoles token provider when using anonymous clients (#21338) Co-authored-by: Lari Hotari --- .../MultiRolesTokenAuthorizationProvider.java | 45 +++++++++++-------- ...tiRolesTokenAuthorizationProviderTest.java | 31 ++++++++++--- 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java index db5f4f18e8cc3..6376b60217fef 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProvider.java @@ -97,7 +97,7 @@ public CompletableFuture isSuperUser(String role, AuthenticationDataSou if (role != null && superUserRoles.contains(role)) { return CompletableFuture.completedFuture(true); } - Set roles = getRoles(authenticationData); + Set roles = getRoles(role, authenticationData); if (roles.isEmpty()) { return CompletableFuture.completedFuture(false); } @@ -112,7 +112,7 @@ public CompletableFuture validateTenantAdminAccess(String tenantName, S if (isSuperUser) { return CompletableFuture.completedFuture(true); } - Set roles = getRoles(authData); + Set roles = getRoles(role, authData); if (roles.isEmpty()) { return CompletableFuture.completedFuture(false); } @@ -143,7 +143,11 @@ public CompletableFuture validateTenantAdminAccess(String tenantName, S }); } - private Set getRoles(AuthenticationDataSource authData) { + private Set getRoles(String role, AuthenticationDataSource authData) { + if (authData == null) { + return Collections.singleton(role); + } + String token = null; if (authData.hasDataFromCommand()) { @@ -192,9 +196,9 @@ private Set getRoles(AuthenticationDataSource authData) { return Collections.emptySet(); } - public CompletableFuture authorize(AuthenticationDataSource authenticationData, Function> authorizeFunc) { - Set roles = getRoles(authenticationData); + public CompletableFuture authorize(String role, AuthenticationDataSource authenticationData, + Function> authorizeFunc) { + Set roles = getRoles(role, authenticationData); if (roles.isEmpty()) { return CompletableFuture.completedFuture(false); } @@ -212,7 +216,7 @@ public CompletableFuture authorize(AuthenticationDataSource authenticat @Override public CompletableFuture canProduceAsync(TopicName topicName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.canProduceAsync(topicName, r, authenticationData)); + return authorize(role, authenticationData, r -> super.canProduceAsync(topicName, r, authenticationData)); } /** @@ -227,7 +231,7 @@ public CompletableFuture canProduceAsync(TopicName topicName, String ro public CompletableFuture canConsumeAsync(TopicName topicName, String role, AuthenticationDataSource authenticationData, String subscription) { - return authorize(authenticationData, r -> super.canConsumeAsync(topicName, r, authenticationData, + return authorize(role, authenticationData, r -> super.canConsumeAsync(topicName, r, authenticationData, subscription)); } @@ -244,25 +248,27 @@ public CompletableFuture canConsumeAsync(TopicName topicName, String ro @Override public CompletableFuture canLookupAsync(TopicName topicName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.canLookupAsync(topicName, r, authenticationData)); + return authorize(role, authenticationData, r -> super.canLookupAsync(topicName, r, authenticationData)); } @Override public CompletableFuture allowFunctionOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.allowFunctionOpsAsync(namespaceName, r, authenticationData)); + return authorize(role, authenticationData, + r -> super.allowFunctionOpsAsync(namespaceName, r, authenticationData)); } @Override public CompletableFuture allowSourceOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.allowSourceOpsAsync(namespaceName, r, authenticationData)); + return authorize(role, authenticationData, + r -> super.allowSourceOpsAsync(namespaceName, r, authenticationData)); } @Override public CompletableFuture allowSinkOpsAsync(NamespaceName namespaceName, String role, AuthenticationDataSource authenticationData) { - return authorize(authenticationData, r -> super.allowSinkOpsAsync(namespaceName, r, authenticationData)); + return authorize(role, authenticationData, r -> super.allowSinkOpsAsync(namespaceName, r, authenticationData)); } @Override @@ -270,7 +276,7 @@ public CompletableFuture allowTenantOperationAsync(String tenantName, String role, TenantOperation operation, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowTenantOperationAsync(tenantName, r, operation, authData)); + return authorize(role, authData, r -> super.allowTenantOperationAsync(tenantName, r, operation, authData)); } @Override @@ -278,7 +284,8 @@ public CompletableFuture allowNamespaceOperationAsync(NamespaceName nam String role, NamespaceOperation operation, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowNamespaceOperationAsync(namespaceName, r, operation, authData)); + return authorize(role, authData, + r -> super.allowNamespaceOperationAsync(namespaceName, r, operation, authData)); } @Override @@ -287,8 +294,8 @@ public CompletableFuture allowNamespacePolicyOperationAsync(NamespaceNa PolicyOperation operation, String role, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowNamespacePolicyOperationAsync(namespaceName, policy, operation, r, - authData)); + return authorize(role, authData, + r -> super.allowNamespacePolicyOperationAsync(namespaceName, policy, operation, r, authData)); } @Override @@ -296,7 +303,7 @@ public CompletableFuture allowTopicOperationAsync(TopicName topicName, String role, TopicOperation operation, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowTopicOperationAsync(topicName, r, operation, authData)); + return authorize(role, authData, r -> super.allowTopicOperationAsync(topicName, r, operation, authData)); } @Override @@ -305,7 +312,7 @@ public CompletableFuture allowTopicPolicyOperationAsync(TopicName topic PolicyName policyName, PolicyOperation policyOperation, AuthenticationDataSource authData) { - return authorize(authData, r -> super.allowTopicPolicyOperationAsync(topicName, r, policyName, policyOperation, - authData)); + return authorize(role, authData, + r -> super.allowTopicPolicyOperationAsync(topicName, r, policyName, policyOperation, authData)); } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java index f0a857bdd695d..4b67f52075ca9 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/authorization/MultiRolesTokenAuthorizationProviderTest.java @@ -24,6 +24,8 @@ import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; import java.util.Properties; +import java.util.function.Function; +import lombok.Cleanup; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; @@ -61,18 +63,18 @@ public String getHttpHeader(String name) { } }; - assertTrue(provider.authorize(ads, role -> { + assertTrue(provider.authorize("test", ads, role -> { if (role.equals(userB)) { return CompletableFuture.completedFuture(true); // only userB has permission } return CompletableFuture.completedFuture(false); }).get()); - assertTrue(provider.authorize(ads, role -> { + assertTrue(provider.authorize("test", ads, role -> { return CompletableFuture.completedFuture(true); // all users has permission }).get()); - assertFalse(provider.authorize(ads, role -> { + assertFalse(provider.authorize("test", ads, role -> { return CompletableFuture.completedFuture(false); // all users has no permission }).get()); } @@ -100,7 +102,7 @@ public String getHttpHeader(String name) { } }; - assertFalse(provider.authorize(ads, role -> CompletableFuture.completedFuture(false)).get()); + assertFalse(provider.authorize("test", ads, role -> CompletableFuture.completedFuture(false)).get()); } @Test @@ -127,7 +129,7 @@ public String getHttpHeader(String name) { } }; - assertTrue(provider.authorize(ads, role -> { + assertTrue(provider.authorize("test", ads, role -> { if (role.equals(testRole)) { return CompletableFuture.completedFuture(true); } @@ -135,6 +137,21 @@ public String getHttpHeader(String name) { }).get()); } + @Test + public void testMultiRolesAuthzWithAnonymousUser() throws Exception { + @Cleanup + MultiRolesTokenAuthorizationProvider provider = new MultiRolesTokenAuthorizationProvider(); + + Function> authorizeFunc = (String role) -> { + if (role.equals("test-role")) { + return CompletableFuture.completedFuture(true); + } + return CompletableFuture.completedFuture(false); + }; + assertTrue(provider.authorize("test-role", null, authorizeFunc).get()); + assertFalse(provider.authorize("test-role-x", null, authorizeFunc).get()); + } + @Test public void testMultiRolesNotFailNonJWT() throws Exception { String token = "a-non-jwt-token"; @@ -157,7 +174,7 @@ public String getHttpHeader(String name) { } }; - assertFalse(provider.authorize(ads, role -> CompletableFuture.completedFuture(false)).get()); + assertFalse(provider.authorize("test", ads, role -> CompletableFuture.completedFuture(false)).get()); } @Test @@ -192,7 +209,7 @@ public String getHttpHeader(String name) { } }; - assertTrue(provider.authorize(ads, role -> { + assertTrue(provider.authorize("test", ads, role -> { if (role.equals(testRole)) { return CompletableFuture.completedFuture(true); } From ff4845b1973cd65c911d2659f91d65fa76b0cccb Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 23 Sep 2023 22:15:23 +0800 Subject: [PATCH 311/494] [fix] [broker] fix flaky test PatternTopicsConsumerImplTest (#21222) (cherry picked from commit be4bcac11ae76cdf3d4c4b0639735f3309919e4c) --- .../impl/PatternTopicsConsumerImplTest.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java index d1c569565bcdd..f01a1a2d07d66 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java @@ -48,6 +48,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.annotations.AfterMethod; @@ -67,6 +68,7 @@ public void setup() throws Exception { isTcpLookup = true; // enabled transaction, to test pattern consumers not subscribe to transaction system topic. conf.setTransactionCoordinatorEnabled(true); + conf.setSubscriptionPatternMaxLength(10000); super.internalSetup(); super.producerBaseSetup(); } @@ -210,6 +212,12 @@ public void testBinaryProtoToGetTopicsOfNamespacePersistent() throws Exception { .subscribe(); assertTrue(consumer.getTopic().startsWith(PatternMultiTopicsConsumerImpl.DUMMY_TOPIC_NAME_PREFIX)); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 4. verify consumer get methods, to get right number of partitions and topics. assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); @@ -287,6 +295,12 @@ public void testBinaryProtoSubscribeAllTopicOfNamespace() throws Exception { .subscribe(); assertTrue(consumer.getTopic().startsWith(PatternMultiTopicsConsumerImpl.DUMMY_TOPIC_NAME_PREFIX)); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 4. verify consumer get methods, to get right number of partitions and topics. assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); @@ -364,6 +378,12 @@ public void testBinaryProtoToGetTopicsOfNamespaceNonPersistent() throws Exceptio .subscriptionTopicsMode(RegexSubscriptionMode.NonPersistentOnly) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 4. verify consumer get methods, to get right number of partitions and topics. assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); @@ -455,6 +475,12 @@ public void testBinaryProtoToGetTopicsOfNamespaceAll() throws Exception { .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 4. verify consumer get methods, to get right number of partitions and topics. assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); List topics = ((PatternMultiTopicsConsumerImpl) consumer).getPartitions(); @@ -525,6 +551,11 @@ public void testStartEmptyPatternConsumer() throws Exception { .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) .receiverQueueSize(4) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); // 3. verify consumer get methods, to get 5 number of partitions and topics. assertSame(pattern, ((PatternMultiTopicsConsumerImpl) consumer).getPattern()); @@ -605,6 +636,12 @@ public void testAutoSubscribePatterConsumerFromBrokerWatcher() throws Exception .receiverQueueSize(4) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + // 1. create partition String topicName = "persistent://my-property/my-ns/pattern-topic-1-" + key; TenantInfoImpl tenantInfo = createDefaultTenantInfo(); @@ -665,6 +702,12 @@ public void testAutoSubscribePatternConsumer() throws Exception { .receiverQueueSize(4) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + assertTrue(consumer instanceof PatternMultiTopicsConsumerImpl); // 4. verify consumer get methods, to get 6 number of partitions and topics: 6=1+2+3 @@ -775,6 +818,12 @@ public void testAutoUnsubscribePatternConsumer() throws Exception { .receiverQueueSize(4) .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + assertTrue(consumer instanceof PatternMultiTopicsConsumerImpl); // 4. verify consumer get methods, to get 0 number of partitions and topics: 6=1+2+3 @@ -861,6 +910,12 @@ public void testTopicDeletion() throws Exception { .subscriptionName("sub") .subscribe(); + // Wait topic list watcher creation. + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + assertTrue(consumer instanceof PatternMultiTopicsConsumerImpl); PatternMultiTopicsConsumerImpl consumerImpl = (PatternMultiTopicsConsumerImpl) consumer; From dd28bb41f44de7d88aec39c79cc8014d64ad4476 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 8 Oct 2023 11:25:02 +0800 Subject: [PATCH 312/494] [fix] [ml] Reader can set read-pos to a deleted ledger (#21248) ### Motivation After trimming ledgers, the variable `lastConfirmedEntry` of the managed ledger might rely on a deleted ledger(the latest ledger which contains data). There is a bug that makes pulsar allow users to set the start read position to an unexisting ledger or a deleted ledger when creating a subscription. This makes the `backlog` and `markDeletedPosition` wrong. ### Modifications Fix the bug. (cherry picked from commit 4ee5cd78147890ce3c23092edbfc11370972728e) --- .../mledger/impl/ManagedCursorImpl.java | 3 + .../mledger/impl/ManagedLedgerImpl.java | 21 +- .../mledger/impl/NonDurableCursorImpl.java | 2 +- .../api/NonDurableSubscriptionTest.java | 216 ++++++++++++++++++ 4 files changed, 234 insertions(+), 8 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index 9063602dd703d..3a4d371019c55 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -1107,6 +1107,9 @@ public long getNumberOfEntriesInBacklog(boolean isPrecise) { messagesConsumedCounter, markDeletePosition, readPosition); } if (isPrecise) { + if (markDeletePosition.compareTo(ledger.getLastPosition()) >= 0) { + return 0; + } return getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition())); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 0f65a1a089c1f..d51b48bdda526 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -3630,23 +3630,30 @@ public PositionImpl getPreviousPosition(PositionImpl position) { * @return true if the position is valid, false otherwise */ public boolean isValidPosition(PositionImpl position) { - PositionImpl last = lastConfirmedEntry; + PositionImpl lac = lastConfirmedEntry; if (log.isDebugEnabled()) { - log.debug("IsValid position: {} -- last: {}", position, last); + log.debug("IsValid position: {} -- last: {}", position, lac); } - if (position.getEntryId() < 0) { + if (!ledgers.containsKey(position.getLedgerId())){ return false; - } else if (position.getLedgerId() > last.getLedgerId()) { + } else if (position.getEntryId() < 0) { return false; - } else if (position.getLedgerId() == last.getLedgerId()) { - return position.getEntryId() <= (last.getEntryId() + 1); + } else if (currentLedger != null && position.getLedgerId() == currentLedger.getId()) { + // If current ledger is empty, the largest read position can be "{current_ledger: 0}". + // Else, the read position can be set to "{LAC + 1}" when subscribe at LATEST, + return (position.getLedgerId() == lac.getLedgerId() && position.getEntryId() <= lac.getEntryId() + 1) + || position.getEntryId() == 0; + } else if (position.getLedgerId() == lac.getLedgerId()) { + // The ledger witch maintains LAC was closed, and there is an empty current ledger. + // If entry id is larger than LAC, it should be "{current_ledger: 0}". + return position.getEntryId() <= lac.getEntryId(); } else { // Look in the ledgers map LedgerInfo ls = ledgers.get(position.getLedgerId()); if (ls == null) { - if (position.getLedgerId() < last.getLedgerId()) { + if (position.getLedgerId() < lac.getLedgerId()) { // Pointing to a non-existing ledger that is older than the current ledger is invalid return false; } else { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java index 51e56158cad55..77216ce2e4588 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorImpl.java @@ -70,7 +70,7 @@ public class NonDurableCursorImpl extends ManagedCursorImpl { private void recoverCursor(PositionImpl mdPosition) { Pair lastEntryAndCounter = ledger.getLastPositionAndCounter(); this.readPosition = isReadCompacted() ? mdPosition.getNext() : ledger.getNextValidPosition(mdPosition); - markDeletePosition = mdPosition; + markDeletePosition = ledger.getPreviousPosition(this.readPosition); // Initialize the counter such that the difference between the messages written on the ML and the // messagesConsumed is equal to the current backlog (negated). diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java index 6762b7d8a045e..223e9f473555d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java @@ -27,6 +27,8 @@ import java.lang.reflect.Field; import java.util.UUID; import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -34,6 +36,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.BrokerService; @@ -45,6 +48,8 @@ import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.client.impl.MessageIdImpl; import org.apache.pulsar.common.api.proto.CommandFlow; +import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats; +import org.apache.pulsar.common.policies.data.SubscriptionStats; import org.awaitility.Awaitility; import org.awaitility.reflect.WhiteboxImpl; import org.testng.Assert; @@ -382,4 +387,215 @@ public void testTrimLedgerIfNoDurableCursor() throws Exception { producer.close(); admin.topics().delete(topicName); } + + @Test + public void testInitReaderAtSpecifiedPosition() throws Exception { + String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, "s0", MessageId.earliest); + + // Trigger 5 ledgers. + ArrayList ledgers = new ArrayList<>(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + for (int i = 0; i < 5; i++) { + MessageIdImpl msgId = (MessageIdImpl) producer.send("1"); + ledgers.add(msgId.getLedgerId()); + admin.topics().unload(topicName); + } + producer.close(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + LedgerHandle currentLedger = WhiteboxImpl.getInternalState(ml, "currentLedger"); + log.info("currentLedger: {}", currentLedger.getId()); + + // Less than the first ledger, and entry id is "-1". + log.info("start test s1"); + String s1 = "s1"; + MessageIdImpl startMessageId1 = new MessageIdImpl(ledgers.get(0) - 1, -1, -1); + Reader reader1 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s1) + .receiverQueueSize(0).startMessageId(startMessageId1).create(); + ManagedLedgerInternalStats.CursorStats cursor1 = admin.topics().getInternalStats(topicName).cursors.get(s1); + log.info("cursor1 readPosition: {}, markDeletedPosition: {}", cursor1.readPosition, cursor1.markDeletePosition); + PositionImpl p1 = parseReadPosition(cursor1); + assertEquals(p1.getLedgerId(), ledgers.get(0)); + assertEquals(p1.getEntryId(), 0); + reader1.close(); + + // Less than the first ledger, and entry id is Long.MAX_VALUE. + log.info("start test s2"); + String s2 = "s2"; + MessageIdImpl startMessageId2 = new MessageIdImpl(ledgers.get(0) - 1, Long.MAX_VALUE, -1); + Reader reader2 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s2) + .receiverQueueSize(0).startMessageId(startMessageId2).create(); + ManagedLedgerInternalStats.CursorStats cursor2 = admin.topics().getInternalStats(topicName).cursors.get(s2); + log.info("cursor2 readPosition: {}, markDeletedPosition: {}", cursor2.readPosition, cursor2.markDeletePosition); + PositionImpl p2 = parseReadPosition(cursor2); + assertEquals(p2.getLedgerId(), ledgers.get(0)); + assertEquals(p2.getEntryId(), 0); + reader2.close(); + + // Larger than the latest ledger, and entry id is "-1". + log.info("start test s3"); + String s3 = "s3"; + MessageIdImpl startMessageId3 = new MessageIdImpl(currentLedger.getId() + 1, -1, -1); + Reader reader3 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s3) + .receiverQueueSize(0).startMessageId(startMessageId3).create(); + ManagedLedgerInternalStats.CursorStats cursor3 = admin.topics().getInternalStats(topicName).cursors.get(s3); + log.info("cursor3 readPosition: {}, markDeletedPosition: {}", cursor3.readPosition, cursor3.markDeletePosition); + PositionImpl p3 = parseReadPosition(cursor3); + assertEquals(p3.getLedgerId(), currentLedger.getId()); + assertEquals(p3.getEntryId(), 0); + reader3.close(); + + // Larger than the latest ledger, and entry id is Long.MAX_VALUE. + log.info("start test s4"); + String s4 = "s4"; + MessageIdImpl startMessageId4 = new MessageIdImpl(currentLedger.getId() + 1, Long.MAX_VALUE, -1); + Reader reader4 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s4) + .receiverQueueSize(0).startMessageId(startMessageId4).create(); + ManagedLedgerInternalStats.CursorStats cursor4 = admin.topics().getInternalStats(topicName).cursors.get(s4); + log.info("cursor4 readPosition: {}, markDeletedPosition: {}", cursor4.readPosition, cursor4.markDeletePosition); + PositionImpl p4 = parseReadPosition(cursor4); + assertEquals(p4.getLedgerId(), currentLedger.getId()); + assertEquals(p4.getEntryId(), 0); + reader4.close(); + + // Ledger id and entry id both are Long.MAX_VALUE. + log.info("start test s5"); + String s5 = "s5"; + MessageIdImpl startMessageId5 = new MessageIdImpl(currentLedger.getId() + 1, Long.MAX_VALUE, -1); + Reader reader5 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s5) + .receiverQueueSize(0).startMessageId(startMessageId5).create(); + ManagedLedgerInternalStats.CursorStats cursor5 = admin.topics().getInternalStats(topicName).cursors.get(s5); + log.info("cursor5 readPosition: {}, markDeletedPosition: {}", cursor5.readPosition, cursor5.markDeletePosition); + PositionImpl p5 = parseReadPosition(cursor5); + assertEquals(p5.getLedgerId(), currentLedger.getId()); + assertEquals(p5.getEntryId(), 0); + reader5.close(); + + // Ledger id equals LAC, and entry id is "-1". + log.info("start test s6"); + String s6 = "s6"; + MessageIdImpl startMessageId6 = new MessageIdImpl(ledgers.get(ledgers.size() - 1), -1, -1); + Reader reader6 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s6) + .receiverQueueSize(0).startMessageId(startMessageId6).create(); + ManagedLedgerInternalStats.CursorStats cursor6 = admin.topics().getInternalStats(topicName).cursors.get(s6); + log.info("cursor6 readPosition: {}, markDeletedPosition: {}", cursor6.readPosition, cursor6.markDeletePosition); + PositionImpl p6 = parseReadPosition(cursor6); + assertEquals(p6.getLedgerId(), ledgers.get(ledgers.size() - 1)); + assertEquals(p6.getEntryId(), 0); + reader6.close(); + + // Larger than the latest ledger, and entry id is Long.MAX_VALUE. + log.info("start test s7"); + String s7 = "s7"; + MessageIdImpl startMessageId7 = new MessageIdImpl(ledgers.get(ledgers.size() - 1), Long.MAX_VALUE, -1); + Reader reader7 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s7) + .receiverQueueSize(0).startMessageId(startMessageId7).create(); + ManagedLedgerInternalStats.CursorStats cursor7 = admin.topics().getInternalStats(topicName).cursors.get(s7); + log.info("cursor7 readPosition: {}, markDeletedPosition: {}", cursor7.readPosition, cursor7.markDeletePosition); + PositionImpl p7 = parseReadPosition(cursor7); + assertEquals(p7.getLedgerId(), currentLedger.getId()); + assertEquals(p7.getEntryId(), 0); + reader7.close(); + + // A middle ledger id, and entry id is "-1". + log.info("start test s8"); + String s8 = "s8"; + MessageIdImpl startMessageId8 = new MessageIdImpl(ledgers.get(2), 0, -1); + Reader reader8 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s8) + .receiverQueueSize(0).startMessageId(startMessageId8).create(); + ManagedLedgerInternalStats.CursorStats cursor8 = admin.topics().getInternalStats(topicName).cursors.get(s8); + log.info("cursor8 readPosition: {}, markDeletedPosition: {}", cursor8.readPosition, cursor8.markDeletePosition); + PositionImpl p8 = parseReadPosition(cursor8); + assertEquals(p8.getLedgerId(), ledgers.get(2)); + assertEquals(p8.getEntryId(), 0); + reader8.close(); + + // Larger than the latest ledger, and entry id is Long.MAX_VALUE. + log.info("start test s9"); + String s9 = "s9"; + MessageIdImpl startMessageId9 = new MessageIdImpl(ledgers.get(2), Long.MAX_VALUE, -1); + Reader reader9 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s9) + .receiverQueueSize(0).startMessageId(startMessageId9).create(); + ManagedLedgerInternalStats.CursorStats cursor9 = admin.topics().getInternalStats(topicName).cursors.get(s9); + log.info("cursor9 readPosition: {}, markDeletedPosition: {}", cursor9.readPosition, + cursor9.markDeletePosition); + PositionImpl p9 = parseReadPosition(cursor9); + assertEquals(p9.getLedgerId(), ledgers.get(3)); + assertEquals(p9.getEntryId(), 0); + reader9.close(); + + // Larger than the latest ledger, and entry id equals with the max entry id of this ledger. + log.info("start test s10"); + String s10 = "s10"; + MessageIdImpl startMessageId10 = new MessageIdImpl(ledgers.get(2), 0, -1); + Reader reader10 = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName(s10) + .receiverQueueSize(0).startMessageId(startMessageId10).create(); + ManagedLedgerInternalStats.CursorStats cursor10 = admin.topics().getInternalStats(topicName).cursors.get(s10); + log.info("cursor10 readPosition: {}, markDeletedPosition: {}", cursor10.readPosition, cursor10.markDeletePosition); + PositionImpl p10 = parseReadPosition(cursor10); + assertEquals(p10.getLedgerId(), ledgers.get(2)); + assertEquals(p10.getEntryId(), 0); + reader10.close(); + + // cleanup + admin.topics().delete(topicName, false); + } + + private PositionImpl parseReadPosition(ManagedLedgerInternalStats.CursorStats cursorStats) { + String[] ledgerIdAndEntryId = cursorStats.readPosition.split(":"); + return PositionImpl.get(Long.valueOf(ledgerIdAndEntryId[0]), Long.valueOf(ledgerIdAndEntryId[1])); + } + + @Test + public void testReaderInitAtDeletedPosition() throws Exception { + String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topicName); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + producer.send("1"); + producer.send("2"); + producer.send("3"); + MessageIdImpl msgIdInDeletedLedger4 = (MessageIdImpl) producer.send("4"); + MessageIdImpl msgIdInDeletedLedger5 = (MessageIdImpl) producer.send("5"); + + // Trigger a trim ledgers task, and verify trim ledgers successful. + admin.topics().unload(topicName); + trimLedgers(topicName); + List ledgers = admin.topics().getInternalStats(topicName).ledgers; + assertEquals(ledgers.size(), 1); + assertNotEquals(ledgers.get(0).ledgerId, msgIdInDeletedLedger5.getLedgerId()); + + // Start a reader at a deleted ledger. + MessageIdImpl startMessageId = + new MessageIdImpl(msgIdInDeletedLedger4.getLedgerId(), msgIdInDeletedLedger4.getEntryId(), -1); + Reader reader = pulsarClient.newReader(Schema.STRING).topic(topicName).subscriptionName("s1") + .startMessageId(startMessageId).create(); + Message msg1 = reader.readNext(2, TimeUnit.SECONDS); + Assert.assertNull(msg1); + + // Verify backlog and markDeletePosition is correct. + Awaitility.await().untilAsserted(() -> { + SubscriptionStats subscriptionStats = admin.topics() + .getStats(topicName, true, true, true).getSubscriptions().get("s1"); + log.info("backlog size: {}", subscriptionStats.getMsgBacklog()); + assertEquals(subscriptionStats.getMsgBacklog(), 0); + ManagedLedgerInternalStats.CursorStats cursorStats = + admin.topics().getInternalStats(topicName).cursors.get("s1"); + String[] ledgerIdAndEntryId = cursorStats.markDeletePosition.split(":"); + PositionImpl actMarkDeletedPos = + PositionImpl.get(Long.valueOf(ledgerIdAndEntryId[0]), Long.valueOf(ledgerIdAndEntryId[1])); + PositionImpl expectedMarkDeletedPos = + PositionImpl.get(msgIdInDeletedLedger5.getLedgerId(), msgIdInDeletedLedger5.getEntryId()); + log.info("Expected mark deleted position: {}", expectedMarkDeletedPos); + log.info("Actual mark deleted position: {}", cursorStats.markDeletePosition); + assertTrue(actMarkDeletedPos.compareTo(expectedMarkDeletedPos) >= 0); + }); + + // cleanup. + reader.close(); + producer.close(); + admin.topics().delete(topicName, false); + } } From 68959190f6b9d5b2ba3468b040cf94acfef87188 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 8 Oct 2023 22:19:12 +0800 Subject: [PATCH 313/494] [fix] [ml] fix wrong msg backlog of non-durable cursor after trim ledgers (#21250) ### Background - But after trimming ledgers, `ml.lastConfirmedPosition` relies on a deleted ledger when the current ledger of ML is empty. - Cursor prevents setting `markDeletedPosition` to a value larger than `ml.lastConfirmedPosition`, but there are no entries to read[1]. - The code description in the method `advanceCursors` said: do not make `cursor.markDeletedPosition` larger than `ml.lastConfirmedPosition`[2] ### Issue If there is no durable cursor, the `markDeletedPosition` might be set to `{current_ledger, -1}`, and `async mark delete` will be prevented by the `rule-2` above. So he `backlog`, `readPosition`, and `markDeletedPosition` of the cursor will be in an incorrect position after trimming the ledger. You can reproduce it by the test `testTrimLedgerIfNoDurableCursor` ### Modifications Do not make `cursor.markDeletedPosition` larger than `ml.lastConfirmedPosition` when advancing non-durable cursors. (cherry picked from commit ca77982758170993aa52c0f7f45bbf9ad72e368a) --- .../mledger/impl/ManagedLedgerImpl.java | 28 +++-- .../api/NonDurableSubscriptionTest.java | 113 +++++++++++++++++- 2 files changed, 132 insertions(+), 9 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index d51b48bdda526..e011bf3e6d73e 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -2850,15 +2850,14 @@ void advanceCursorsIfNecessary(List ledgersToDelete) throws LedgerNo return; } - // need to move mark delete for non-durable cursors to the first ledger NOT marked for deletion - // calling getNumberOfEntries latter for a ledger that is already deleted will be problematic and return - // incorrect results - Long firstNonDeletedLedger = ledgers.higherKey(ledgersToDelete.get(ledgersToDelete.size() - 1).getLedgerId()); - if (firstNonDeletedLedger == null) { - throw new LedgerNotExistException("First non deleted Ledger is not found"); + // Just ack messages like a consumer. Normally, consumers will not confirm a position that does not exist, so + // find the latest existing position to ack. + PositionImpl highestPositionToDelete = calculateLastEntryInLedgerList(ledgersToDelete); + if (highestPositionToDelete == null) { + log.warn("[{}] The ledgers to be trim are all empty, skip to advance non-durable cursors: {}", + name, ledgersToDelete); + return; } - PositionImpl highestPositionToDelete = new PositionImpl(firstNonDeletedLedger, -1); - cursors.forEach(cursor -> { // move the mark delete position to the highestPositionToDelete only if it is smaller than the add confirmed // to prevent the edge case where the cursor is caught up to the latest and highestPositionToDelete may be @@ -2882,6 +2881,19 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) { }); } + /** + * @return null if all ledgers is empty. + */ + private PositionImpl calculateLastEntryInLedgerList(List ledgersToDelete) { + for (int i = ledgersToDelete.size() - 1; i >= 0; i--) { + LedgerInfo ledgerInfo = ledgersToDelete.get(i); + if (ledgerInfo != null && ledgerInfo.hasEntries() && ledgerInfo.getEntries() > 0) { + return PositionImpl.get(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1); + } + } + return null; + } + /** * Delete this ManagedLedger completely from the system. * diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java index 223e9f473555d..f9c259a97473c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonDurableSubscriptionTest.java @@ -36,6 +36,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; @@ -357,7 +358,7 @@ private void switchLedgerManually(final String tpName) throws Exception { } @Test - public void testTrimLedgerIfNoDurableCursor() throws Exception { + public void testHasMessageAvailableIfIncomingQueueNotEmpty() throws Exception { final String nonDurableCursor = "non-durable-cursor"; final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); Reader reader = pulsarClient.newReader(Schema.STRING).topic(topicName).receiverQueueSize(1) @@ -598,4 +599,114 @@ public void testReaderInitAtDeletedPosition() throws Exception { producer.close(); admin.topics().delete(topicName, false); } + + @Test + public void testTrimLedgerIfNoDurableCursor() throws Exception { + final String nonDurableCursor = "non-durable-cursor"; + final String durableCursor = "durable-cursor"; + final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(topicName); + Reader reader = pulsarClient.newReader(Schema.STRING).topic(topicName).receiverQueueSize(1) + .subscriptionName(nonDurableCursor).startMessageId(MessageIdImpl.earliest).create(); + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topicName).receiverQueueSize(1) + .subscriptionName(durableCursor).subscribe(); + consumer.close(); + + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topicName).create(); + producer.send("1"); + producer.send("2"); + producer.send("3"); + producer.send("4"); + MessageIdImpl msgIdInDeletedLedger5 = (MessageIdImpl) producer.send("5"); + + Message msg1 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg1.getValue(), "1"); + Message msg2 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg2.getValue(), "2"); + Message msg3 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg3.getValue(), "3"); + + // Unsubscribe durable cursor. + // Trigger a trim ledgers task, and verify trim ledgers successful. + admin.topics().unload(topicName); + Thread.sleep(3 * 1000); + admin.topics().deleteSubscription(topicName, durableCursor); + // Trim ledgers after release durable cursor. + trimLedgers(topicName); + List ledgers = admin.topics().getInternalStats(topicName).ledgers; + assertEquals(ledgers.size(), 1); + assertNotEquals(ledgers.get(0).ledgerId, msgIdInDeletedLedger5.getLedgerId()); + + // Verify backlog and markDeletePosition is correct. + Awaitility.await().untilAsserted(() -> { + SubscriptionStats subscriptionStats = admin.topics().getStats(topicName, true, true, true) + .getSubscriptions().get(nonDurableCursor); + log.info("backlog size: {}", subscriptionStats.getMsgBacklog()); + assertEquals(subscriptionStats.getMsgBacklog(), 0); + ManagedLedgerInternalStats.CursorStats cursorStats = + admin.topics().getInternalStats(topicName).cursors.get(nonDurableCursor); + String[] ledgerIdAndEntryId = cursorStats.markDeletePosition.split(":"); + PositionImpl actMarkDeletedPos = + PositionImpl.get(Long.valueOf(ledgerIdAndEntryId[0]), Long.valueOf(ledgerIdAndEntryId[1])); + PositionImpl expectedMarkDeletedPos = + PositionImpl.get(msgIdInDeletedLedger5.getLedgerId(), msgIdInDeletedLedger5.getEntryId()); + log.info("Expected mark deleted position: {}", expectedMarkDeletedPos); + log.info("Actual mark deleted position: {}", cursorStats.markDeletePosition); + Assert.assertTrue(actMarkDeletedPos.compareTo(expectedMarkDeletedPos) >= 0); + }); + + // Clear the incoming queue of the reader for next test. + while (true) { + Message msg = reader.readNext(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + log.info("clear msg: {}", msg.getValue()); + } + + // The following tests are designed to verify the api "getNumberOfEntries" and "consumedEntries" still work + // after changes.See the code-description added with the PR https://github.com/apache/pulsar/pull/10667. + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + ManagedLedgerImpl ml = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().get(nonDurableCursor); + + // Verify "getNumberOfEntries" if there is no entries to consume. + assertEquals(0, cursor.getNumberOfEntries()); + assertEquals(0, ml.getNumberOfEntries()); + + // Verify "getNumberOfEntries" if there is 1 entry to consume. + producer.send("6"); + producer.send("7"); + Awaitility.await().untilAsserted(() -> { + assertEquals(2, ml.getNumberOfEntries()); + // Since there is one message has been pulled into the incoming queue of reader. There is only one messages + // waiting to cursor read. + assertEquals(1, cursor.getNumberOfEntries()); + }); + + // Verify "consumedEntries" is correct. + ManagedLedgerInternalStats.CursorStats cursorStats = + admin.topics().getInternalStats(topicName).cursors.get(nonDurableCursor); + // "messagesConsumedCounter" should be 0 after unload the topic. + // Note: "topic_internal_stat.cursor.messagesConsumedCounter" means how many messages were acked on this + // cursor. The similar one "topic_stats.lastConsumedTimestamp" means the last time of sending messages to + // the consumer. + assertEquals(0, cursorStats.messagesConsumedCounter); + Message msg6 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg6.getValue(), "6"); + Message msg7 = reader.readNext(2, TimeUnit.SECONDS); + assertEquals(msg7.getValue(), "7"); + Awaitility.await().untilAsserted(() -> { + // "messagesConsumedCounter" should be 2 after consumed 2 message. + ManagedLedgerInternalStats.CursorStats cStat = + admin.topics().getInternalStats(topicName).cursors.get(nonDurableCursor); + assertEquals(2, cStat.messagesConsumedCounter); + }); + + // cleanup. + reader.close(); + producer.close(); + admin.topics().delete(topicName, false); + } } From 3727a6915e2fe682477a47c6f4abc23db7ab5761 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 14 Oct 2023 10:45:10 +0300 Subject: [PATCH 314/494] [fix][ci] Fix docker image building by releasing more disk space before building (#21365) (cherry picked from commit 421c98a1a5ec08941e794698dfc43a1a08d6e782) --- .github/actions/clean-disk/action.yml | 2 +- .github/workflows/pulsar-ci.yaml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/actions/clean-disk/action.yml b/.github/actions/clean-disk/action.yml index 8bcc5f1396802..d74c3f25fc64c 100644 --- a/.github/actions/clean-disk/action.yml +++ b/.github/actions/clean-disk/action.yml @@ -31,7 +31,7 @@ runs: directories=(/usr/local/lib/android /opt/ghc) if [[ "${{ inputs.mode }}" == "full" ]]; then # remove these directories only when mode is 'full' - directories+=(/usr/share/dotnet) + directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL) fi emptydir=/tmp/empty$$/ mkdir $emptydir diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 35182f0bbd837..5f8267bf54860 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -747,6 +747,8 @@ jobs: - name: Clean Disk uses: ./.github/actions/clean-disk + with: + mode: full - name: Cache local Maven repository uses: actions/cache@v3 @@ -862,6 +864,7 @@ jobs: - name: Pulsar IO group: PULSAR_IO + clean_disk: true - name: Sql group: SQL @@ -873,6 +876,10 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm + - name: Clean Disk when needed + if: ${{ matrix.clean_disk }} + uses: ./.github/actions/clean-disk + - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} @@ -1073,6 +1080,7 @@ jobs: - name: Pulsar IO - Oracle group: PULSAR_IO_ORA + clean_disk: true steps: - name: checkout @@ -1081,6 +1089,10 @@ jobs: - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm + - name: Clean Disk when needed + if: ${{ matrix.clean_disk }} + uses: ./.github/actions/clean-disk + - name: Setup ssh access to build runner VM # ssh access is enabled for builds in own forks if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} From c89d267cbe148c08a24d3989851a218d2738f73a Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Wed, 11 Oct 2023 11:07:50 +0800 Subject: [PATCH 315/494] [fix] [bk-client] Fix bk client MinNumRacksPerWriteQuorum and EnforceMinNumRacksPerWriteQuorum not work problem. (#21327) (cherry picked from commit 61a7adf08b14067500f9bd17b6da824ba58e9707) --- .../broker/BookKeeperClientFactoryImpl.java | 5 ++++- .../BookKeeperClientFactoryImplTest.java | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java index 0259dfc7a58cf..1674930778d09 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/BookKeeperClientFactoryImpl.java @@ -220,7 +220,7 @@ static void setDefaultEnsemblePlacementPolicy( } } - private void setEnsemblePlacementPolicy(ClientConfiguration bkConf, ServiceConfiguration conf, MetadataStore store, + static void setEnsemblePlacementPolicy(ClientConfiguration bkConf, ServiceConfiguration conf, MetadataStore store, Class policyClass) { bkConf.setEnsemblePlacementPolicy(policyClass); bkConf.setProperty(BookieRackAffinityMapping.METADATA_STORE_INSTANCE, store); @@ -228,6 +228,9 @@ private void setEnsemblePlacementPolicy(ClientConfiguration bkConf, ServiceConfi bkConf.setProperty(REPP_DNS_RESOLVER_CLASS, conf.getProperties().getProperty(REPP_DNS_RESOLVER_CLASS, BookieRackAffinityMapping.class.getName())); + bkConf.setMinNumRacksPerWriteQuorum(conf.getBookkeeperClientMinNumRacksPerWriteQuorum()); + bkConf.setEnforceMinNumRacksPerWriteQuorum(conf.isBookkeeperClientEnforceMinNumRacksPerWriteQuorum()); + bkConf.setProperty(NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, conf.getProperties().getProperty( NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java index a02689dc9763a..9ff18baf95912 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/BookKeeperClientFactoryImplTest.java @@ -41,6 +41,7 @@ import org.apache.pulsar.bookie.rackawareness.BookieRackAffinityMapping; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.zookeeper.ZkIsolatedBookieEnsemblePlacementPolicy; import org.testng.annotations.Test; /** @@ -152,6 +153,24 @@ public void testSetDefaultEnsemblePlacementPolicyRackAwareEnabledChangedValues() assertEquals(20, bkConf.getMinNumRacksPerWriteQuorum()); } + @Test + public void testSetEnsemblePlacementPolicys() { + ClientConfiguration bkConf = new ClientConfiguration(); + ServiceConfiguration conf = new ServiceConfiguration(); + conf.setBookkeeperClientMinNumRacksPerWriteQuorum(3); + conf.setBookkeeperClientEnforceMinNumRacksPerWriteQuorum(true); + + MetadataStore store = mock(MetadataStore.class); + + BookKeeperClientFactoryImpl.setEnsemblePlacementPolicy( + bkConf, + conf, + store, + ZkIsolatedBookieEnsemblePlacementPolicy.class); + assertEquals(bkConf.getMinNumRacksPerWriteQuorum(), 3); + assertTrue(bkConf.getEnforceMinNumRacksPerWriteQuorum()); + } + @Test public void testSetDiskWeightBasedPlacementEnabled() { BookKeeperClientFactoryImpl factory = new BookKeeperClientFactoryImpl(); From 9dd81af2aa9f169a04901cdc407a6739fd0c9f93 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 16 Oct 2023 15:47:16 +0300 Subject: [PATCH 316/494] [fix][test] Fix a resource leak in ClusterMigrationTest (#21366) (cherry picked from commit 39235edcb45a615627dcd0471ed0872e568790ff) --- .../broker/service/ClusterMigrationTest.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java index df4f66c43d2b4..69af6d2f28d6d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java @@ -25,11 +25,11 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; - +import com.google.common.collect.Sets; import java.lang.reflect.Method; import java.net.URL; import java.util.concurrent.TimeUnit; - +import lombok.Cleanup; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -50,10 +50,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.google.common.collect.Sets; - -import lombok.Cleanup; - @Test(groups = "broker") public class ClusterMigrationTest { @@ -202,9 +198,13 @@ public void setup() throws Exception { protected void cleanup() throws Exception { log.info("--- Shutting down ---"); broker1.cleanup(); + admin1.close(); broker2.cleanup(); + admin2.close(); broker3.cleanup(); + admin3.close(); broker4.cleanup(); + admin4.close(); } @BeforeMethod(alwaysRun = true) @@ -399,7 +399,7 @@ public void testClusterMigrationWithReplicationBacklog(boolean persistent, Subsc assertEquals(topic1.getReplicators().size(), 1); // stop service in the replication cluster to build replication backlog - broker3.cleanup(); + broker3.stop(); retryStrategically((test) -> broker3.getPulsarService() == null, 10, 1000); assertNull(pulsar3.getBrokerService()); @@ -485,9 +485,13 @@ public String getClusterName() { return configClusterName; } + public void stop() throws Exception { + stopBroker(); + } + @Override protected void cleanup() throws Exception { - stopBroker(); + internalCleanup(); } public void restart() throws Exception { From d7ca04bf5e94e660829ccf1efb0e9e0964f9120c Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 9 Oct 2023 12:07:01 +0800 Subject: [PATCH 317/494] [feat][sql] Support UUID for json and avro (#21267) ### Motivation As https://pulsar.apache.org/docs/3.1.x/sql-overview/, Pulsar SQL is based on [Trino (formerly Presto SQL)](https://trino.io/), which supports UUID type. But now, the UUID field in Avro or JSON schema will be interpreted as VARCHAR. ### Modifications Support decoding UUID form AVRO or JSON schema. (cherry picked from commit 8c7094328e03b11bf57e8f9d1022047961b75481) --- .../sql/presto/decoder/avro/PulsarAvroColumnDecoder.java | 9 ++++++++- .../presto/decoder/avro/PulsarAvroRowDecoderFactory.java | 5 +++++ .../sql/presto/decoder/json/PulsarJsonFieldDecoder.java | 4 +++- .../presto/decoder/json/PulsarJsonRowDecoderFactory.java | 5 +++++ .../pulsar/sql/presto/decoder/DecoderTestMessage.java | 4 ++++ .../pulsar/sql/presto/decoder/avro/TestAvroDecoder.java | 7 +++++++ .../pulsar/sql/presto/decoder/json/TestJsonDecoder.java | 8 ++++++++ 7 files changed, 40 insertions(+), 2 deletions(-) diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java index 73081f8948a51..1672d5f144817 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroColumnDecoder.java @@ -54,6 +54,7 @@ import io.trino.spi.type.Timestamps; import io.trino.spi.type.TinyintType; import io.trino.spi.type.Type; +import io.trino.spi.type.UuidType; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; import java.math.BigInteger; @@ -61,6 +62,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import org.apache.avro.generic.GenericEnumSymbol; import org.apache.avro.generic.GenericFixed; import org.apache.avro.generic.GenericRecord; @@ -87,7 +89,8 @@ public class PulsarAvroColumnDecoder { TimestampType.TIMESTAMP_MILLIS, DateType.DATE, TimeType.TIME_MILLIS, - VarbinaryType.VARBINARY); + VarbinaryType.VARBINARY, + UuidType.UUID); private final Type columnType; private final String columnMapping; @@ -255,6 +258,10 @@ private static Slice getSlice(Object value, Type type, String columnName) { } } + if (type instanceof UuidType) { + return UuidType.javaUuidToTrinoUuid(UUID.fromString(value.toString())); + } + throw new TrinoException(DECODER_CONVERSION_NOT_SUPPORTED, format("cannot decode object of '%s' as '%s' for column '%s'", value.getClass(), type, columnName)); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java index 3072bf9441b2c..e6eb6b7f2f947 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/avro/PulsarAvroRowDecoderFactory.java @@ -44,6 +44,7 @@ import io.trino.spi.type.TypeManager; import io.trino.spi.type.TypeSignature; import io.trino.spi.type.TypeSignatureParameter; +import io.trino.spi.type.UuidType; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; import java.util.List; @@ -121,6 +122,10 @@ private Type parseAvroPrestoType(String fieldName, Schema schema) { LogicalType logicalType = schema.getLogicalType(); switch (type) { case STRING: + if (logicalType != null && logicalType.equals(LogicalTypes.uuid())) { + return UuidType.UUID; + } + return createUnboundedVarcharType(); case ENUM: return createUnboundedVarcharType(); case NULL: diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java index 905e3bd6becb4..8e744e3b1229c 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonFieldDecoder.java @@ -58,6 +58,7 @@ import io.trino.spi.type.Timestamps; import io.trino.spi.type.TinyintType; import io.trino.spi.type.Type; +import io.trino.spi.type.UuidType; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; import java.util.Iterator; @@ -126,7 +127,8 @@ private boolean isSupportedType(Type type) { TimestampType.TIMESTAMP_MILLIS, DateType.DATE, TimeType.TIME_MILLIS, - RealType.REAL + RealType.REAL, + UuidType.UUID ).contains(type)) { return true; } diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java index 0d5cc2d262dfe..737eb608d82d6 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/decoder/json/PulsarJsonRowDecoderFactory.java @@ -44,6 +44,7 @@ import io.trino.spi.type.TypeManager; import io.trino.spi.type.TypeSignature; import io.trino.spi.type.TypeSignatureParameter; +import io.trino.spi.type.UuidType; import io.trino.spi.type.VarbinaryType; import io.trino.spi.type.VarcharType; import java.util.List; @@ -121,6 +122,10 @@ private Type parseJsonPrestoType(String fieldName, Schema schema) { LogicalType logicalType = schema.getLogicalType(); switch (type) { case STRING: + if (logicalType != null && logicalType.equals(LogicalTypes.uuid())) { + return UuidType.UUID; + } + return createUnboundedVarcharType(); case ENUM: return createUnboundedVarcharType(); case NULL: diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java index 4561282c67196..0dec76b3d4dec 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/DecoderTestMessage.java @@ -19,6 +19,7 @@ package org.apache.pulsar.sql.presto.decoder; import java.math.BigDecimal; +import java.util.UUID; import lombok.Data; import java.util.List; @@ -55,6 +56,9 @@ public static enum TestEnum { public Map mapField; public CompositeRow compositeRow; + @org.apache.avro.reflect.AvroSchema("{\"type\":\"string\",\"logicalType\":\"uuid\"}") + public UUID uuidField; + public static class TestRow { public String stringField; public int intField; diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java index c4e7009b9465b..5f9df96619b9f 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/avro/TestAvroDecoder.java @@ -44,6 +44,7 @@ import io.trino.spi.type.Timestamps; import io.trino.spi.type.Type; import io.trino.spi.type.TypeSignatureParameter; +import io.trino.spi.type.UuidType; import io.trino.spi.type.VarcharType; import java.math.BigDecimal; import java.time.LocalDate; @@ -55,6 +56,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.generic.GenericAvroRecord; import org.apache.pulsar.client.impl.schema.generic.GenericAvroSchema; @@ -90,6 +92,7 @@ public void testPrimitiveType() { message.longField = 222L; message.timestampField = System.currentTimeMillis(); message.enumField = DecoderTestMessage.TestEnum.TEST_ENUM_1; + message.uuidField = UUID.randomUUID(); LocalTime now = LocalTime.now(ZoneId.systemDefault()); message.timeField = now.toSecondOfDay() * 1000; @@ -137,6 +140,10 @@ public void testPrimitiveType() { PulsarColumnHandle timeFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "timeField", TIME_MILLIS, false, false, "timeField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); checkValue(decodedRow, timeFieldColumnHandle, (long) message.timeField * Timestamps.PICOSECONDS_PER_MILLISECOND); + + PulsarColumnHandle uuidHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "uuidField", UuidType.UUID, false, false, "uuidField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, uuidHandle, UuidType.javaUuidToTrinoUuid(message.uuidField)); } @Test diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java index 4afad9b318fc5..32e71a53444cf 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/decoder/json/TestJsonDecoder.java @@ -44,6 +44,7 @@ import io.trino.spi.type.Timestamps; import io.trino.spi.type.Type; import io.trino.spi.type.TypeSignatureParameter; +import io.trino.spi.type.UuidType; import io.trino.spi.type.VarcharType; import java.math.BigDecimal; import java.time.LocalDate; @@ -55,6 +56,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.UUID; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.generic.GenericJsonRecord; import org.apache.pulsar.client.impl.schema.generic.GenericJsonSchema; @@ -98,6 +100,8 @@ public void testPrimitiveType() { LocalDate epoch = LocalDate.ofEpochDay(0); message.dateField = Math.toIntExact(ChronoUnit.DAYS.between(epoch, localDate)); + message.uuidField = UUID.randomUUID(); + ByteBuf payload = io.netty.buffer.Unpooled .copiedBuffer(schema.encode(message)); Map decodedRow = pulsarRowDecoder.decodeRow(payload).get(); @@ -137,6 +141,10 @@ public void testPrimitiveType() { PulsarColumnHandle timeFieldColumnHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), "timeField", TIME_MILLIS, false, false, "timeField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); checkValue(decodedRow, timeFieldColumnHandle, (long) message.timeField * Timestamps.PICOSECONDS_PER_MILLISECOND); + + PulsarColumnHandle uuidHandle = new PulsarColumnHandle(getPulsarConnectorId().toString(), + "uuidField", UuidType.UUID, false, false, "uuidField", null, null, PulsarColumnHandle.HandleKeyValueType.NONE); + checkValue(decodedRow, uuidHandle, message.uuidField.toString()); } @Test From 215f421041fb47da7e9406b65108dec9154f60c0 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 17 Oct 2023 13:17:58 +0300 Subject: [PATCH 318/494] [fix][test] Fix some resource leaks in compaction tests (#21374) (cherry picked from commit e6cd11fb49f2d8f3376f2218e60f8c5909f66348) # Conflicts: # pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java --- .../java/org/apache/pulsar/compaction/CompactedTopicTest.java | 2 ++ .../pulsar/compaction/ServiceUnitStateCompactionTest.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java index 957671b7f8d9c..8524eb27aacce 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactedTopicTest.java @@ -164,6 +164,7 @@ public void cleanup() throws Exception { @Test public void testEntryLookup() throws Exception { + @Cleanup BookKeeper bk = pulsar.getBookKeeperClientFactory().create( this.conf, null, null, Optional.empty(), null); @@ -219,6 +220,7 @@ public void testEntryLookup() throws Exception { @Test public void testCleanupOldCompactedTopicLedger() throws Exception { + @Cleanup BookKeeper bk = pulsar.getBookKeeperClientFactory().create( this.conf, null, null, Optional.empty(), null); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java index e4f0750a981c9..09c3ebe419394 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/ServiceUnitStateCompactionTest.java @@ -168,7 +168,7 @@ public void setup() throws Exception { @Override public void cleanup() throws Exception { super.internalCleanup(); - + bk.close(); if (compactionScheduler != null) { compactionScheduler.shutdownNow(); } From d35618a6ddbb6fe0bc6babb17463beef70339612 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 17 Oct 2023 15:34:16 +0300 Subject: [PATCH 319/494] [fix][test] Fix resource leaks with Pulsar Functions tests (#21378) (cherry picked from commit ffc083b5ea9da998c35161b9dcfbfb6e38e3917e) --- .../pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java | 1 - .../pulsar/functions/worker/PulsarFunctionPublishTest.java | 1 - .../pulsar/functions/worker/PulsarWorkerAssignmentTest.java | 1 - .../test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java | 1 - .../test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java | 1 - 5 files changed, 5 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java index 714c9d7269970..107aedd076691 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java @@ -268,7 +268,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthorizationProvider(config.getAuthorizationProvider()); PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java index 95923586fe23e..6fa7172773cd8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java @@ -263,7 +263,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthorizationEnabled(true); PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java index 0821974bea506..6226fa904885c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java @@ -174,7 +174,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setTopicCompactionFrequencySec(1); PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java index 9991e9f1b70fd..19de771a56853 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java @@ -301,7 +301,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthorizationEnabled(true); PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java index 16218f6ce6448..ec17382062cf1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarFunctionAdminTest.java @@ -207,7 +207,6 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH); PulsarWorkerService workerService = new PulsarWorkerService(); - workerService.init(workerConfig, null, false); return workerService; } } From af6449d5b113b8bd6ddc528ddee840ffc42472f0 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Wed, 18 Oct 2023 07:55:15 -0500 Subject: [PATCH 320/494] [fix] [auto-recovery] Fix PulsarLedgerUnderreplicationManager notify problem. (#21312) --- pulsar-metadata/pom.xml | 2 - .../PulsarLedgerUnderreplicationManager.java | 64 ++++++++++++------- .../replication/AuditorLedgerCheckerTest.java | 15 +++-- .../LedgerUnderreplicationManagerTest.java | 19 +++++- 4 files changed, 67 insertions(+), 33 deletions(-) diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 3bf76b8336783..97881803b5db4 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -57,7 +57,6 @@ - io.dropwizard.metrics metrics-core @@ -85,7 +84,6 @@ - ${project.groupId} testmocks diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java index dda8d7256ed52..343c3165ec7e9 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerUnderreplicationManager.java @@ -102,14 +102,17 @@ long getLedgerNodeVersion() { private final String urLockPath; private final String layoutPath; private final String lostBookieRecoveryDelayPath; + private final String replicationDisablePath; private final String checkAllLedgersCtimePath; private final String placementPolicyCheckCtimePath; private final String replicasCheckCtimePath; private final MetadataStoreExtended store; - private BookkeeperInternalCallbacks.GenericCallback replicationEnabledListener; - private BookkeeperInternalCallbacks.GenericCallback lostBookieRecoveryDelayListener; + private final List> replicationEnabledCallbacks = + new ArrayList<>(); + private final List> lostBookieRecoveryDelayCallbacks = + new ArrayList<>(); private static class PulsarUnderreplicatedLedger extends UnderreplicatedLedger { PulsarUnderreplicatedLedger(long ledgerId) { @@ -136,6 +139,7 @@ public PulsarLedgerUnderreplicationManager(AbstractConfiguration conf, Metada urLedgerPath = basePath + BookKeeperConstants.DEFAULT_ZK_LEDGERS_ROOT_PATH; urLockPath = basePath + '/' + BookKeeperConstants.UNDER_REPLICATION_LOCK; lostBookieRecoveryDelayPath = basePath + '/' + BookKeeperConstants.LOSTBOOKIERECOVERYDELAY_NODE; + replicationDisablePath = basePath + '/' + BookKeeperConstants.DISABLE_NODE; checkAllLedgersCtimePath = basePath + '/' + BookKeeperConstants.CHECK_ALL_LEDGERS_CTIME; placementPolicyCheckCtimePath = basePath + '/' + BookKeeperConstants.PLACEMENT_POLICY_CHECK_CTIME; replicasCheckCtimePath = basePath + '/' + BookKeeperConstants.REPLICAS_CHECK_CTIME; @@ -229,17 +233,34 @@ private void handleNotification(Notification n) { synchronized (this) { // Notify that there were some changes on the under-replicated z-nodes notifyAll(); - - if (n.getType() == NotificationType.Deleted) { - if (n.getPath().equals(basePath + '/' + BookKeeperConstants.DISABLE_NODE)) { - log.info("LedgerReplication is enabled externally through MetadataStore, " - + "since DISABLE_NODE ZNode is deleted"); - if (replicationEnabledListener != null) { - replicationEnabledListener.operationComplete(0, null); + if (lostBookieRecoveryDelayPath.equals(n.getPath())) { + final List> callbackList; + synchronized (lostBookieRecoveryDelayCallbacks) { + callbackList = new ArrayList<>(lostBookieRecoveryDelayCallbacks); + lostBookieRecoveryDelayCallbacks.clear(); + } + for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { + try { + callback.operationComplete(0, null); + } catch (Exception e) { + log.warn("lostBookieRecoveryDelayCallbacks handle error", e); } - } else if (n.getPath().equals(lostBookieRecoveryDelayPath)) { - if (lostBookieRecoveryDelayListener != null) { - lostBookieRecoveryDelayListener.operationComplete(0, null); + } + return; + } + if (replicationDisablePath.equals(n.getPath()) && n.getType() == NotificationType.Deleted) { + log.info("LedgerReplication is enabled externally through MetadataStore, " + + "since DISABLE_NODE ZNode is deleted"); + final List> callbackList; + synchronized (replicationEnabledCallbacks) { + callbackList = new ArrayList<>(replicationEnabledCallbacks); + replicationEnabledCallbacks.clear(); + } + for (BookkeeperInternalCallbacks.GenericCallback callback : callbackList) { + try { + callback.operationComplete(0, null); + } catch (Exception e) { + log.warn("replicationEnabledCallbacks handle error", e); } } } @@ -671,8 +692,7 @@ public void disableLedgerReplication() log.debug("disableLedegerReplication()"); } try { - String path = basePath + '/' + BookKeeperConstants.DISABLE_NODE; - store.put(path, "".getBytes(UTF_8), Optional.of(-1L)).get(); + store.put(replicationDisablePath, "".getBytes(UTF_8), Optional.of(-1L)).get(); log.info("Auto ledger re-replication is disabled!"); } catch (ExecutionException ee) { log.error("Exception while stopping auto ledger re-replication", ee); @@ -692,7 +712,7 @@ public void enableLedgerReplication() log.debug("enableLedegerReplication()"); } try { - store.delete(basePath + '/' + BookKeeperConstants.DISABLE_NODE, Optional.empty()).get(); + store.delete(replicationDisablePath, Optional.empty()).get(); log.info("Resuming automatic ledger re-replication"); } catch (ExecutionException ee) { log.error("Exception while resuming ledger replication", ee); @@ -712,7 +732,7 @@ public boolean isLedgerReplicationEnabled() log.debug("isLedgerReplicationEnabled()"); } try { - return !store.exists(basePath + '/' + BookKeeperConstants.DISABLE_NODE).get(); + return !store.exists(replicationDisablePath).get(); } catch (ExecutionException ee) { log.error("Error while checking the state of " + "ledger re-replication", ee); @@ -731,13 +751,11 @@ public void notifyLedgerReplicationEnabled(final BookkeeperInternalCallbacks.Gen if (log.isDebugEnabled()) { log.debug("notifyLedgerReplicationEnabled()"); } - - synchronized (this) { - replicationEnabledListener = cb; + synchronized (replicationEnabledCallbacks) { + replicationEnabledCallbacks.add(cb); } - try { - if (!store.exists(basePath + '/' + BookKeeperConstants.DISABLE_NODE).get()) { + if (!store.exists(replicationDisablePath).get()) { log.info("LedgerReplication is enabled externally through metadata store, " + "since DISABLE_NODE node is deleted"); cb.operationComplete(0, null); @@ -826,8 +844,8 @@ public int getLostBookieRecoveryDelay() throws ReplicationException.UnavailableE public void notifyLostBookieRecoveryDelayChanged(BookkeeperInternalCallbacks.GenericCallback cb) throws ReplicationException.UnavailableException { log.debug("notifyLostBookieRecoveryDelayChanged()"); - synchronized (this) { - lostBookieRecoveryDelayListener = cb; + synchronized (lostBookieRecoveryDelayCallbacks) { + lostBookieRecoveryDelayCallbacks.add(cb); } try { if (!store.exists(lostBookieRecoveryDelayPath).get()) { diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index ec5f77f79464b..12bc6fbe55d24 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -23,10 +23,10 @@ import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertNotSame; import static org.testng.AssertJUnit.assertTrue; -import static org.testng.AssertJUnit.fail; -import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; +import static org.testng.AssertJUnit.fail; +import java.net.URI; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -40,15 +40,20 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import java.util.concurrent.TimeoutException; import lombok.Cleanup; import org.apache.bookkeeper.bookie.BookieImpl; import org.apache.bookkeeper.client.AsyncCallback.AddCallback; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.client.LedgerMetadataBuilder; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.common.util.OrderedScheduler; @@ -60,11 +65,7 @@ import org.apache.bookkeeper.meta.MetadataClientDriver; import org.apache.bookkeeper.meta.MetadataDrivers; import org.apache.bookkeeper.meta.UnderreplicatedLedger; -import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; -import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; -import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; -import org.apache.bookkeeper.proto.BookieServer; import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.pulsar.metadata.api.MetadataStoreConfig; diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java index 649dc1663c68f..0e9c781fb9143 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/LedgerUnderreplicationManagerTest.java @@ -40,6 +40,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import lombok.Cleanup; @@ -614,6 +615,8 @@ public void testDisableLedgerReplication(String provider, Supplier urlSu final String missingReplica = "localhost:3181"; // disabling replication + AtomicInteger callbackCount = new AtomicInteger(); + lum.notifyLedgerReplicationEnabled((rc, result) -> callbackCount.incrementAndGet()); lum.disableLedgerReplication(); log.info("Disabled Ledeger Replication"); @@ -631,6 +634,7 @@ public void testDisableLedgerReplication(String provider, Supplier urlSu } catch (TimeoutException te) { // expected behaviour, as the replication is disabled } + assertEquals(callbackCount.get(), 1, "Notify callback times mismatch"); } /** @@ -651,7 +655,8 @@ public void testEnableLedgerReplication(String provider, Supplier urlSup log.debug("Unexpected exception while marking urLedger", e); fail("Unexpected exception while marking urLedger" + e.getMessage()); } - + AtomicInteger callbackCount = new AtomicInteger(); + lum.notifyLedgerReplicationEnabled((rc, result) -> callbackCount.incrementAndGet()); // disabling replication lum.disableLedgerReplication(); log.debug("Disabled Ledeger Replication"); @@ -688,6 +693,7 @@ public void testEnableLedgerReplication(String provider, Supplier urlSup znodeLatch.await(5, TimeUnit.SECONDS); log.debug("Enabled Ledeger Replication"); assertEquals(znodeLatch.getCount(), 0, "Failed to disable ledger replication!"); + assertEquals(callbackCount.get(), 2, "Notify callback times mismatch"); } finally { thread1.interrupt(); } @@ -749,6 +755,17 @@ public void testReplicasCheckCTime(String provider, Supplier urlSupplier assertEquals(underReplicaMgr1.getReplicasCheckCTime(), curTime); } + @Test(timeOut = 60000, dataProvider = "impl") + public void testLostBookieRecoveryDelay(String provider, Supplier urlSupplier) throws Exception { + methodSetup(urlSupplier); + + AtomicInteger callbackCount = new AtomicInteger(); + lum.notifyLostBookieRecoveryDelayChanged((rc, result) -> callbackCount.incrementAndGet()); + // disabling replication + lum.setLostBookieRecoveryDelay(10); + Awaitility.await().until(() -> callbackCount.get() == 2); + } + private void verifyMarkLedgerUnderreplicated(Collection missingReplica) throws Exception { Long ledgerA = 0xfeadeefdacL; String znodeA = getUrLedgerZnode(ledgerA); From 10affd2b17415186844f2dfa46044eb0d78e5972 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 18 Oct 2023 18:21:03 +0300 Subject: [PATCH 321/494] [improve][ci] Add new CI unit test group "Broker Group 4" with cluster migration tests (#21391) (cherry picked from commit e2c6c08b3598e01aba1bac90b56412e451140385) --- .github/workflows/pulsar-ci.yaml | 2 ++ build/run_unit_group.sh | 4 ++++ .../apache/pulsar/broker/service/ClusterMigrationTest.java | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 5f8267bf54860..eaa173e4f135e 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -190,6 +190,8 @@ jobs: group: BROKER_GROUP_2 - name: Brokers - Broker Group 3 group: BROKER_GROUP_3 + - name: Brokers - Broker Group 4 + group: BROKER_GROUP_4 - name: Brokers - Client Api group: BROKER_CLIENT_API - name: Brokers - Client Impl diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 69434b011b37e..17d0efeed9937 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -87,6 +87,10 @@ function test_group_broker_group_3() { mvn_test -pl pulsar-broker -Dgroups='broker-admin' } +function test_group_broker_group_4() { + mvn_test -pl pulsar-broker -Dgroups='cluster-migration' +} + function test_group_broker_client_api() { mvn_test -pl pulsar-broker -Dgroups='broker-api' } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java index 69af6d2f28d6d..cfa9a5f9b0461 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java @@ -50,7 +50,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -@Test(groups = "broker") +@Test(groups = "cluster-migration") public class ClusterMigrationTest { private static final Logger log = LoggerFactory.getLogger(ClusterMigrationTest.class); From f26589ef638faad3eaaf6aba8f6e8d03074025b7 Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 19 Oct 2023 09:07:56 -0500 Subject: [PATCH 322/494] [fix][broker] Fix lookup heartbeat and sla namespace bundle when using extensible load manager (#21213) (#21313) --- .../apache/pulsar/broker/PulsarService.java | 2 +- .../pulsar/broker/loadbalance/LoadData.java | 2 +- .../extensions/ExtensibleLoadManagerImpl.java | 338 +++++++++++------- .../channel/ServiceUnitStateChannelImpl.java | 71 ++-- .../extensions/models/TopKBundles.java | 5 +- .../broker/namespace/NamespaceService.java | 41 +-- .../ExtensibleLoadManagerImplTest.java | 70 +++- .../channel/ServiceUnitStateChannelTest.java | 57 +-- 8 files changed, 321 insertions(+), 265 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 6bb477cf03723..87e1a17b9b0a3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1165,7 +1165,7 @@ protected void startLeaderElectionService() { protected void acquireSLANamespace() { try { // Namespace not created hence no need to unload it - NamespaceName nsName = NamespaceService.getSLAMonitorNamespace(getAdvertisedAddress(), config); + NamespaceName nsName = NamespaceService.getSLAMonitorNamespace(getLookupServiceAddress(), config); if (!this.pulsarResources.getNamespaceResources().namespaceExists(nsName)) { LOG.info("SLA Namespace = {} doesn't exist.", nsName); return; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java index a632a47f05116..c1fe2a4930c34 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/LoadData.java @@ -64,7 +64,7 @@ public Map getBundleData() { public Map getBundleDataForLoadShedding() { return bundleData.entrySet().stream() - .filter(e -> !NamespaceService.filterNamespaceForShedding( + .filter(e -> !NamespaceService.isSLAOrHeartbeatNamespace( NamespaceBundle.getBundleNamespace(e.getKey()))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index cba499eb8eedb..85baf9ec4fbdf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -44,6 +44,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -86,6 +87,7 @@ import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; +import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; @@ -95,7 +97,6 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; import org.slf4j.Logger; @@ -152,6 +153,7 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { @Getter private final List brokerFilterPipeline; + /** * The load data reporter. */ @@ -181,10 +183,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { // record split metrics private final AtomicReference> splitMetrics = new AtomicReference<>(); - private final ConcurrentOpenHashMap>> - lookupRequests = ConcurrentOpenHashMap.>>newBuilder() - .build(); + private final ConcurrentHashMap>> + lookupRequests = new ConcurrentHashMap<>(); private final CountDownLatch initWaiter = new CountDownLatch(1); /** @@ -197,7 +197,7 @@ public Set getOwnedServiceUnits() { } Set> entrySet = serviceUnitStateChannel.getOwnershipEntrySet(); String brokerId = brokerRegistry.getBrokerId(); - return entrySet.stream() + Set ownedServiceUnits = entrySet.stream() .filter(entry -> { var stateData = entry.getValue(); return stateData.state() == ServiceUnitState.Owned @@ -207,6 +207,36 @@ public Set getOwnedServiceUnits() { var bundle = entry.getKey(); return getNamespaceBundle(pulsar, bundle); }).collect(Collectors.toSet()); + // Add heartbeat and SLA monitor namespace bundle. + NamespaceName heartbeatNamespace = NamespaceService.getHeartbeatNamespace(brokerId, pulsar.getConfiguration()); + try { + NamespaceBundle fullBundle = pulsar.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespace); + ownedServiceUnits.add(fullBundle); + } catch (Exception e) { + log.warn("Failed to get heartbeat namespace bundle.", e); + } + NamespaceName heartbeatNamespaceV2 = NamespaceService + .getHeartbeatNamespaceV2(brokerId, pulsar.getConfiguration()); + try { + NamespaceBundle fullBundle = pulsar.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(heartbeatNamespaceV2); + ownedServiceUnits.add(fullBundle); + } catch (Exception e) { + log.warn("Failed to get heartbeat namespace V2 bundle.", e); + } + + NamespaceName slaMonitorNamespace = NamespaceService + .getSLAMonitorNamespace(brokerId, pulsar.getConfiguration()); + try { + NamespaceBundle fullBundle = pulsar.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(slaMonitorNamespace); + ownedServiceUnits.add(fullBundle); + } catch (Exception e) { + log.warn("Failed to get SLA Monitor namespace bundle.", e); + } + + return ownedServiceUnits; } public enum Role { @@ -261,102 +291,108 @@ public void start() throws PulsarServerException { if (this.started) { return; } - this.brokerRegistry = new BrokerRegistryImpl(pulsar); - this.leaderElectionService = new LeaderElectionService( - pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), ELECTION_ROOT, - state -> { - pulsar.getLoadManagerExecutor().execute(() -> { - if (state == LeaderElectionState.Leading) { - playLeader(); - } else { - playFollower(); - } + try { + this.brokerRegistry = new BrokerRegistryImpl(pulsar); + this.leaderElectionService = new LeaderElectionService( + pulsar.getCoordinationService(), pulsar.getSafeWebServiceAddress(), ELECTION_ROOT, + state -> { + pulsar.getLoadManagerExecutor().execute(() -> { + if (state == LeaderElectionState.Leading) { + playLeader(); + } else { + playFollower(); + } + }); }); - }); - this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar); - this.brokerRegistry.start(); - this.splitManager = new SplitManager(splitCounter); - this.unloadManager = new UnloadManager(unloadCounter); - this.serviceUnitStateChannel.listen(unloadManager); - this.serviceUnitStateChannel.listen(splitManager); - this.leaderElectionService.start(); - this.serviceUnitStateChannel.start(); - this.antiAffinityGroupPolicyHelper = - new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel); - antiAffinityGroupPolicyHelper.listenFailureDomainUpdate(); - this.antiAffinityGroupPolicyFilter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper); - this.brokerFilterPipeline.add(antiAffinityGroupPolicyFilter); - SimpleResourceAllocationPolicies policies = new SimpleResourceAllocationPolicies(pulsar); - this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies); - this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper)); - - createSystemTopic(pulsar, BROKER_LOAD_DATA_STORE_TOPIC); - createSystemTopic(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); + this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar); + this.brokerRegistry.start(); + this.splitManager = new SplitManager(splitCounter); + this.unloadManager = new UnloadManager(unloadCounter); + this.serviceUnitStateChannel.listen(unloadManager); + this.serviceUnitStateChannel.listen(splitManager); + this.leaderElectionService.start(); + this.serviceUnitStateChannel.start(); + this.antiAffinityGroupPolicyHelper = + new AntiAffinityGroupPolicyHelper(pulsar, serviceUnitStateChannel); + antiAffinityGroupPolicyHelper.listenFailureDomainUpdate(); + this.antiAffinityGroupPolicyFilter = new AntiAffinityGroupPolicyFilter(antiAffinityGroupPolicyHelper); + this.brokerFilterPipeline.add(antiAffinityGroupPolicyFilter); + SimpleResourceAllocationPolicies policies = new SimpleResourceAllocationPolicies(pulsar); + this.isolationPoliciesHelper = new IsolationPoliciesHelper(policies); + this.brokerFilterPipeline.add(new BrokerIsolationPoliciesFilter(isolationPoliciesHelper)); + + createSystemTopic(pulsar, BROKER_LOAD_DATA_STORE_TOPIC); + createSystemTopic(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); - try { - this.brokerLoadDataStore = LoadDataStoreFactory - .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); - this.brokerLoadDataStore.startTableView(); - this.topBundlesLoadDataStore = LoadDataStoreFactory - .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); - } catch (LoadDataStoreException e) { - throw new PulsarServerException(e); - } + try { + this.brokerLoadDataStore = LoadDataStoreFactory + .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); + this.brokerLoadDataStore.startTableView(); + this.topBundlesLoadDataStore = LoadDataStoreFactory + .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); + } catch (LoadDataStoreException e) { + throw new PulsarServerException(e); + } - this.context = LoadManagerContextImpl.builder() - .configuration(conf) - .brokerRegistry(brokerRegistry) - .brokerLoadDataStore(brokerLoadDataStore) - .topBundleLoadDataStore(topBundlesLoadDataStore).build(); - - this.brokerLoadDataReporter = - new BrokerLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), brokerLoadDataStore); - - this.topBundleLoadDataReporter = - new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore); - this.serviceUnitStateChannel.listen(brokerLoadDataReporter); - this.serviceUnitStateChannel.listen(topBundleLoadDataReporter); - var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis(); - this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor() - .scheduleAtFixedRate(() -> { - try { - brokerLoadDataReporter.reportAsync(false); - // TODO: update broker load metrics using getLocalData - } catch (Throwable e) { - log.error("Failed to run the broker load manager executor job.", e); - } - }, - interval, - interval, TimeUnit.MILLISECONDS); - - this.topBundlesLoadDataReportTask = this.pulsar.getLoadManagerExecutor() - .scheduleAtFixedRate(() -> { - try { - // TODO: consider excluding the bundles that are in the process of split. - topBundleLoadDataReporter.reportAsync(false); - } catch (Throwable e) { - log.error("Failed to run the top bundles load manager executor job.", e); - } - }, - interval, - interval, TimeUnit.MILLISECONDS); - - this.monitorTask = this.pulsar.getLoadManagerExecutor() - .scheduleAtFixedRate(() -> { - monitor(); - }, - MONITOR_INTERVAL_IN_MILLIS, - MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS); - - this.unloadScheduler = new UnloadScheduler( - pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, - serviceUnitStateChannel, unloadCounter, unloadMetrics); - this.unloadScheduler.start(); - this.splitScheduler = new SplitScheduler( - pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context); - this.splitScheduler.start(); - this.initWaiter.countDown(); - this.started = true; + this.context = LoadManagerContextImpl.builder() + .configuration(conf) + .brokerRegistry(brokerRegistry) + .brokerLoadDataStore(brokerLoadDataStore) + .topBundleLoadDataStore(topBundlesLoadDataStore).build(); + + this.brokerLoadDataReporter = + new BrokerLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), brokerLoadDataStore); + + this.topBundleLoadDataReporter = + new TopBundleLoadDataReporter(pulsar, brokerRegistry.getBrokerId(), topBundlesLoadDataStore); + this.serviceUnitStateChannel.listen(brokerLoadDataReporter); + this.serviceUnitStateChannel.listen(topBundleLoadDataReporter); + var interval = conf.getLoadBalancerReportUpdateMinIntervalMillis(); + this.brokerLoadDataReportTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + try { + brokerLoadDataReporter.reportAsync(false); + // TODO: update broker load metrics using getLocalData + } catch (Throwable e) { + log.error("Failed to run the broker load manager executor job.", e); + } + }, + interval, + interval, TimeUnit.MILLISECONDS); + + this.topBundlesLoadDataReportTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + try { + // TODO: consider excluding the bundles that are in the process of split. + topBundleLoadDataReporter.reportAsync(false); + } catch (Throwable e) { + log.error("Failed to run the top bundles load manager executor job.", e); + } + }, + interval, + interval, TimeUnit.MILLISECONDS); + + this.monitorTask = this.pulsar.getLoadManagerExecutor() + .scheduleAtFixedRate(() -> { + monitor(); + }, + MONITOR_INTERVAL_IN_MILLIS, + MONITOR_INTERVAL_IN_MILLIS, TimeUnit.MILLISECONDS); + + this.unloadScheduler = new UnloadScheduler( + pulsar, pulsar.getLoadManagerExecutor(), unloadManager, context, + serviceUnitStateChannel, unloadCounter, unloadMetrics); + this.unloadScheduler.start(); + this.splitScheduler = new SplitScheduler( + pulsar, serviceUnitStateChannel, splitManager, splitCounter, splitMetrics, context); + this.splitScheduler.start(); + this.initWaiter.countDown(); + this.started = true; + } catch (Exception ex) { + if (this.brokerRegistry != null) { + brokerRegistry.close(); + } + } } @Override @@ -377,25 +413,38 @@ public CompletableFuture> assign(Optional getOwnerAsync( - ServiceUnitId serviceUnit, String bundle, boolean ownByLocalBrokerIfAbsent) { + private String getHeartbeatOrSLAMonitorBrokerId(ServiceUnitId serviceUnit) { + // Check if this is Heartbeat or SLAMonitor namespace + String candidateBroker = NamespaceService.checkHeartbeatNamespace(serviceUnit); + if (candidateBroker == null) { + candidateBroker = NamespaceService.checkHeartbeatNamespaceV2(serviceUnit); + } + if (candidateBroker == null) { + candidateBroker = NamespaceService.getSLAMonitorBrokerName(serviceUnit); + } + if (candidateBroker != null) { + return candidateBroker.substring(candidateBroker.lastIndexOf('/') + 1); + } + return candidateBroker; + } + + private CompletableFuture getOrSelectOwnerAsync(ServiceUnitId serviceUnit, + String bundle) { return serviceUnitStateChannel.getOwnerAsync(bundle).thenCompose(broker -> { // If the bundle not assign yet, select and publish assign event to channel. if (broker.isEmpty()) { - CompletableFuture> selectedBroker; - if (ownByLocalBrokerIfAbsent) { - String brokerId = this.brokerRegistry.getBrokerId(); - selectedBroker = CompletableFuture.completedFuture(Optional.of(brokerId)); - } else { - selectedBroker = this.selectAsync(serviceUnit); - } - return selectedBroker.thenCompose(brokerOpt -> { + return this.selectAsync(serviceUnit).thenCompose(brokerOpt -> { if (brokerOpt.isPresent()) { assignCounter.incrementSuccess(); log.info("Selected new owner broker: {} for bundle: {}.", brokerOpt.get(), bundle); @@ -425,7 +474,8 @@ private CompletableFuture> getBrokerLookupData( }).thenCompose(broker -> this.getBrokerRegistry().lookupAsync(broker).thenCompose(brokerLookupData -> { if (brokerLookupData.isEmpty()) { String errorMsg = String.format( - "Failed to look up a broker registry:%s for bundle:%s", broker, bundle); + "Failed to lookup broker:%s for bundle:%s, the broker has not been registered.", + broker, bundle); log.error(errorMsg); throw new IllegalStateException(errorMsg); } @@ -443,30 +493,37 @@ private CompletableFuture> getBrokerLookupData( public CompletableFuture tryAcquiringOwnership(NamespaceBundle namespaceBundle) { log.info("Try acquiring ownership for bundle: {} - {}.", namespaceBundle, brokerRegistry.getBrokerId()); final String bundle = namespaceBundle.toString(); - return dedupeLookupRequest(bundle, k -> { - final CompletableFuture owner = - this.getOwnerAsync(namespaceBundle, bundle, true); - return getBrokerLookupData(owner.thenApply(Optional::ofNullable), bundle); - }).thenApply(brokerLookupData -> { - if (brokerLookupData.isEmpty()) { - throw new IllegalStateException( - "Failed to get the broker lookup data for bundle: " + bundle); - } - return brokerLookupData.get().toNamespaceEphemeralData(); - }); + return assign(Optional.empty(), namespaceBundle) + .thenApply(brokerLookupData -> { + if (brokerLookupData.isEmpty()) { + String errorMsg = String.format( + "Failed to get the broker lookup data for bundle:%s", bundle); + log.error(errorMsg); + throw new IllegalStateException(errorMsg); + } + return brokerLookupData.get().toNamespaceEphemeralData(); + }); } private CompletableFuture> dedupeLookupRequest( String key, Function>> provider) { - CompletableFuture> future = lookupRequests.computeIfAbsent(key, provider); - future.whenComplete((r, t) -> { - if (t != null) { + final MutableObject>> newFutureCreated = new MutableObject<>(); + try { + return lookupRequests.computeIfAbsent(key, k -> { + CompletableFuture> future = provider.apply(k); + newFutureCreated.setValue(future); + return future; + }); + } finally { + if (newFutureCreated.getValue() != null) { + newFutureCreated.getValue().whenComplete((v, ex) -> { + if (ex != null) { assignCounter.incrementFailure(); } - lookupRequests.remove(key); - } - ); - return future; + lookupRequests.remove(key, newFutureCreated.getValue()); + }); + } + } } public CompletableFuture> selectAsync(ServiceUnitId bundle) { @@ -521,15 +578,16 @@ public CompletableFuture checkOwnershipAsync(Optional to } public CompletableFuture> getOwnershipAsync(Optional topic, - ServiceUnitId bundleUnit) { - final String bundle = bundleUnit.toString(); - CompletableFuture> owner; + ServiceUnitId serviceUnit) { + final String bundle = serviceUnit.toString(); if (topic.isPresent() && isInternalTopic(topic.get().toString())) { - owner = serviceUnitStateChannel.getChannelOwnerAsync(); - } else { - owner = serviceUnitStateChannel.getOwnerAsync(bundle); + return serviceUnitStateChannel.getChannelOwnerAsync(); } - return owner; + String candidateBroker = getHeartbeatOrSLAMonitorBrokerId(serviceUnit); + if (candidateBroker != null) { + return CompletableFuture.completedFuture(Optional.of(candidateBroker)); + } + return serviceUnitStateChannel.getOwnerAsync(bundle); } public CompletableFuture> getOwnershipWithLookupDataAsync(ServiceUnitId bundleUnit) { @@ -543,6 +601,10 @@ public CompletableFuture> getOwnershipWithLookupDataA public CompletableFuture unloadNamespaceBundleAsync(ServiceUnitId bundle, Optional destinationBroker) { + if (NamespaceService.isSLAOrHeartbeatNamespace(bundle.getNamespaceObject().toString())) { + log.info("Skip unloading namespace bundle: {}.", bundle); + return CompletableFuture.completedFuture(null); + } return getOwnershipAsync(Optional.empty(), bundle) .thenCompose(brokerOpt -> { if (brokerOpt.isEmpty()) { @@ -577,6 +639,10 @@ private CompletableFuture unloadAsync(UnloadDecision unloadDecision, public CompletableFuture splitNamespaceBundleAsync(ServiceUnitId bundle, NamespaceBundleSplitAlgorithm splitAlgorithm, List boundaries) { + if (NamespaceService.isSLAOrHeartbeatNamespace(bundle.getNamespaceObject().toString())) { + log.info("Skip split namespace bundle: {}.", bundle); + return CompletableFuture.completedFuture(null); + } final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle.toString()); final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle.toString()); NamespaceBundle namespaceBundle = diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index e9fbf625edd7b..bb5b587219133 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -41,8 +41,6 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Stable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MetadataState.Unstable; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; -import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_FMT; -import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_FMT_V2; import static org.apache.pulsar.common.naming.NamespaceName.SYSTEM_NAMESPACE; import static org.apache.pulsar.common.topics.TopicCompactionStrategy.TABLE_VIEW_TAG; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; @@ -94,7 +92,6 @@ import org.apache.pulsar.common.naming.NamespaceBundleFactory; import org.apache.pulsar.common.naming.NamespaceBundleSplitAlgorithm; import org.apache.pulsar.common.naming.NamespaceBundles; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.stats.Metrics; @@ -1215,48 +1212,19 @@ private synchronized void doCleanup(String broker) { log.info("Started ownership cleanup for the inactive broker:{}", broker); int orphanServiceUnitCleanupCnt = 0; long totalCleanupErrorCntStart = totalCleanupErrorCnt.get(); - String heartbeatNamespace = - NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT, config.getClusterName(), broker)).toString(); - String heartbeatNamespaceV2 = - NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT_V2, broker)).toString(); - Map orphanSystemServiceUnits = new HashMap<>(); for (var etr : tableview.entrySet()) { var stateData = etr.getValue(); var serviceUnit = etr.getKey(); var state = state(stateData); - if (StringUtils.equals(broker, stateData.dstBroker())) { - if (isActiveState(state)) { - if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { - orphanSystemServiceUnits.put(serviceUnit, stateData); - } else if (serviceUnit.startsWith(heartbeatNamespace) - || serviceUnit.startsWith(heartbeatNamespaceV2)) { - // Skip the heartbeat namespace - log.info("Skip override heartbeat namespace bundle" - + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); - tombstoneAsync(serviceUnit).whenComplete((__, e) -> { - if (e != null) { - log.error("Failed cleaning the heartbeat namespace ownership serviceUnit:{}, " - + "stateData:{}, cleanupErrorCnt:{}.", - serviceUnit, stateData, - totalCleanupErrorCnt.incrementAndGet() - totalCleanupErrorCntStart, e); - } - }); - } else { - overrideOwnership(serviceUnit, stateData, broker); - } - orphanServiceUnitCleanupCnt++; - } - - } else if (StringUtils.equals(broker, stateData.sourceBroker())) { - if (isInFlightState(state)) { - if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { - orphanSystemServiceUnits.put(serviceUnit, stateData); - } else { - overrideOwnership(serviceUnit, stateData, broker); - } - orphanServiceUnitCleanupCnt++; + if (StringUtils.equals(broker, stateData.dstBroker()) && isActiveState(state) + || StringUtils.equals(broker, stateData.sourceBroker()) && isInFlightState(state)) { + if (serviceUnit.startsWith(SYSTEM_NAMESPACE.toString())) { + orphanSystemServiceUnits.put(serviceUnit, stateData); + } else { + overrideOwnership(serviceUnit, stateData, broker); } + orphanServiceUnitCleanupCnt++; } } @@ -1400,16 +1368,21 @@ protected void monitorOwnerships(List brokers) { String srcBroker = stateData.sourceBroker(); var state = stateData.state(); - if (isActiveState(state)) { - if (StringUtils.isNotBlank(srcBroker) && !activeBrokers.contains(srcBroker)) { - inactiveBrokers.add(srcBroker); - } else if (StringUtils.isNotBlank(dstBroker) && !activeBrokers.contains(dstBroker)) { - inactiveBrokers.add(dstBroker); - } else if (isInFlightState(state) - && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { - orphanServiceUnits.put(serviceUnit, stateData); - } - } else if (now - stateData.timestamp() > semiTerminalStateWaitingTimeInMillis) { + if (isActiveState(state) && StringUtils.isNotBlank(srcBroker) && !activeBrokers.contains(srcBroker)) { + inactiveBrokers.add(srcBroker); + continue; + } + if (isActiveState(state) && StringUtils.isNotBlank(dstBroker) && !activeBrokers.contains(dstBroker)) { + inactiveBrokers.add(dstBroker); + continue; + } + if (isActiveState(state) && isInFlightState(state) + && now - stateData.timestamp() > inFlightStateWaitingTimeInMillis) { + orphanServiceUnits.put(serviceUnit, stateData); + continue; + } + + if (now - stateData.timestamp() > semiTerminalStateWaitingTimeInMillis) { log.info("Found semi-terminal states to tombstone" + " serviceUnit:{}, stateData:{}", serviceUnit, stateData); tombstoneAsync(serviceUnit).whenComplete((__, e) -> { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java index 2f5c32197c1fd..624546fdff837 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/models/TopKBundles.java @@ -30,6 +30,8 @@ import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; @@ -70,7 +72,8 @@ public void update(Map bundleStats, int topk) { pulsar.getConfiguration().isLoadBalancerSheddingBundlesWithPoliciesEnabled(); for (var etr : bundleStats.entrySet()) { String bundle = etr.getKey(); - if (bundle.startsWith(NamespaceName.SYSTEM_NAMESPACE.toString())) { + // TODO: do not filter system topic while shedding + if (NamespaceService.isSystemServiceNamespace(NamespaceBundle.getBundleNamespace(bundle))) { continue; } if (!isLoadBalancerSheddingBundlesWithPoliciesEnabled && hasPolicies(bundle)) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index abf963614089d..0d69fd7ea7801 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -134,7 +134,7 @@ public class NamespaceService implements AutoCloseable { public static final Pattern SLA_NAMESPACE_PATTERN = Pattern.compile(SLA_NAMESPACE_PROPERTY + "/[^/]+/([^:]+:\\d+)"); public static final String HEARTBEAT_NAMESPACE_FMT = "pulsar/%s/%s"; public static final String HEARTBEAT_NAMESPACE_FMT_V2 = "pulsar/%s"; - public static final String SLA_NAMESPACE_FMT = SLA_NAMESPACE_PROPERTY + "/%s/%s:%s"; + public static final String SLA_NAMESPACE_FMT = SLA_NAMESPACE_PROPERTY + "/%s/%s"; private final ConcurrentOpenHashMap namespaceClients; @@ -183,7 +183,7 @@ public CompletableFuture> getBrokerServiceUrlAsync(TopicN CompletableFuture> future = getBundleAsync(topic) .thenCompose(bundle -> { // Do redirection if the cluster is in rollback or deploying. - return redirectManager.findRedirectLookupResultAsync().thenCompose(optResult -> { + return findRedirectLookupResultAsync(bundle).thenCompose(optResult -> { if (optResult.isPresent()) { LOG.info("[{}] Redirect lookup request to {} for topic {}", pulsar.getSafeWebServiceAddress(), optResult.get(), topic); @@ -215,6 +215,13 @@ public CompletableFuture> getBrokerServiceUrlAsync(TopicN return future; } + private CompletableFuture> findRedirectLookupResultAsync(ServiceUnitId bundle) { + if (isSLAOrHeartbeatNamespace(bundle.getNamespaceObject().toString())) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return redirectManager.findRedirectLookupResultAsync(); + } + public CompletableFuture getBundleAsync(TopicName topic) { return bundleFactory.getBundlesAsync(topic.getNamespaceObject()) .thenApply(bundles -> bundles.findBundle(topic)); @@ -283,8 +290,7 @@ public Optional getWebServiceUrl(ServiceUnitId suName, LookupOptions option private CompletableFuture> internalGetWebServiceUrl(Optional topic, NamespaceBundle bundle, LookupOptions options) { - - return redirectManager.findRedirectLookupResultAsync().thenCompose(optResult -> { + return findRedirectLookupResultAsync(bundle).thenCompose(optResult -> { if (optResult.isPresent()) { LOG.info("[{}] Redirect lookup request to {} for topic {}", pulsar.getSafeWebServiceAddress(), optResult.get(), topic); @@ -698,7 +704,7 @@ public CompletableFuture createLookupResult(String candidateBroker return lookupFuture; } - private boolean isBrokerActive(String candidateBroker) { + public boolean isBrokerActive(String candidateBroker) { String candidateBrokerHostAndPort = parseHostAndPort(candidateBroker); Set availableBrokers = getAvailableBrokers(); if (availableBrokers.contains(candidateBrokerHostAndPort)) { @@ -1550,8 +1556,7 @@ public CompletableFuture checkOwnershipPresentAsync(NamespaceBundle bun } public void unloadSLANamespace() throws Exception { - PulsarAdmin adminClient = null; - NamespaceName namespaceName = getSLAMonitorNamespace(host, config); + NamespaceName namespaceName = getSLAMonitorNamespace(pulsar.getLookupServiceAddress(), config); LOG.info("Checking owner for SLA namespace {}", namespaceName); @@ -1563,7 +1568,7 @@ public void unloadSLANamespace() throws Exception { } LOG.info("Trying to unload SLA namespace {}", namespaceName); - adminClient = pulsar.getAdminClient(); + PulsarAdmin adminClient = pulsar.getAdminClient(); adminClient.namespaces().unload(namespaceName.toString()); LOG.info("Namespace {} unloaded successfully", namespaceName); } @@ -1576,14 +1581,8 @@ public static NamespaceName getHeartbeatNamespaceV2(String lookupBroker, Service return NamespaceName.get(String.format(HEARTBEAT_NAMESPACE_FMT_V2, lookupBroker)); } - public static NamespaceName getSLAMonitorNamespace(String host, ServiceConfiguration config) { - Integer port = null; - if (config.getWebServicePort().isPresent()) { - port = config.getWebServicePort().get(); - } else if (config.getWebServicePortTls().isPresent()) { - port = config.getWebServicePortTls().get(); - } - return NamespaceName.get(String.format(SLA_NAMESPACE_FMT, config.getClusterName(), host, port)); + public static NamespaceName getSLAMonitorNamespace(String lookupBroker, ServiceConfiguration config) { + return NamespaceName.get(String.format(SLA_NAMESPACE_FMT, config.getClusterName(), lookupBroker)); } public static String checkHeartbeatNamespace(ServiceUnitId ns) { @@ -1627,7 +1626,7 @@ public static boolean isSystemServiceNamespace(String namespace) { * @param namespace * @return True if namespace is HEARTBEAT_NAMESPACE or SLA_NAMESPACE */ - public static boolean filterNamespaceForShedding(String namespace) { + public static boolean isSLAOrHeartbeatNamespace(String namespace) { return SLA_NAMESPACE_PATTERN.matcher(namespace).matches() || HEARTBEAT_NAMESPACE_PATTERN.matcher(namespace).matches() || HEARTBEAT_NAMESPACE_PATTERN_V2.matcher(namespace).matches(); @@ -1640,14 +1639,16 @@ public static boolean isHeartbeatNamespace(ServiceUnitId ns) { } public boolean registerSLANamespace() throws PulsarServerException { - boolean isNameSpaceRegistered = registerNamespace(getSLAMonitorNamespace(host, config), false); + String lookupServiceAddress = pulsar.getLookupServiceAddress(); + boolean isNameSpaceRegistered = registerNamespace(getSLAMonitorNamespace(lookupServiceAddress, config), false); if (isNameSpaceRegistered) { if (LOG.isDebugEnabled()) { LOG.debug("Added SLA Monitoring namespace name in local cache: ns={}", - getSLAMonitorNamespace(host, config)); + getSLAMonitorNamespace(lookupServiceAddress, config)); } } else if (LOG.isDebugEnabled()) { - LOG.debug("SLA Monitoring not owned by the broker: ns={}", getSLAMonitorNamespace(host, config)); + LOG.debug("SLA Monitoring not owned by the broker: ns={}", + getSLAMonitorNamespace(lookupServiceAddress, config)); } return isNameSpaceRegistered; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 76202b1b0ae30..2d9de4b5e7f07 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -35,6 +35,9 @@ import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Overloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Underloaded; import static org.apache.pulsar.broker.loadbalance.extensions.models.UnloadDecision.Reason.Unknown; +import static org.apache.pulsar.broker.namespace.NamespaceService.getHeartbeatNamespace; +import static org.apache.pulsar.broker.namespace.NamespaceService.getHeartbeatNamespaceV2; +import static org.apache.pulsar.broker.namespace.NamespaceService.getSLAMonitorNamespace; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -575,6 +578,18 @@ public void testDeployAndRollbackLoadManager() throws Exception { assertTrue(webServiceUrl3.isPresent()); assertEquals(webServiceUrl3.get().toString(), webServiceUrl1.get().toString()); + List pulsarServices = List.of(pulsar1, pulsar2, pulsar3); + for (PulsarService pulsarService : pulsarServices) { + // Test lookup heartbeat namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupHeartbeatOwner(pulsarService, pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + } + // Test lookup SLA namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupSLANamespaceOwner(pulsarService, pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + } + } + // Test deploy new broker with new load manager ServiceConfiguration conf = getDefaultConf(); conf.setAllowAutoTopicCreation(true); @@ -623,10 +638,48 @@ public void testDeployAndRollbackLoadManager() throws Exception { assertTrue(webServiceUrl4.isPresent()); assertEquals(webServiceUrl4.get().toString(), webServiceUrl1.get().toString()); + pulsarServices = List.of(pulsar1, pulsar2, pulsar3, pulsar4); + for (PulsarService pulsarService : pulsarServices) { + // Test lookup heartbeat namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupHeartbeatOwner(pulsarService, pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + } + // Test lookup SLA namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupSLANamespaceOwner(pulsarService, pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + } + } } } } + private void assertLookupHeartbeatOwner(PulsarService pulsar, + String lookupServiceAddress, + String expectedBrokerServiceUrl) throws Exception { + NamespaceName heartbeatNamespaceV1 = + getHeartbeatNamespace(lookupServiceAddress, pulsar.getConfiguration()); + + String heartbeatV1Topic = heartbeatNamespaceV1.getPersistentTopicName("test"); + assertEquals(pulsar.getAdminClient().lookups().lookupTopic(heartbeatV1Topic), expectedBrokerServiceUrl); + + NamespaceName heartbeatNamespaceV2 = + getHeartbeatNamespaceV2(lookupServiceAddress, pulsar.getConfiguration()); + + String heartbeatV2Topic = heartbeatNamespaceV2.getPersistentTopicName("test"); + assertEquals(pulsar.getAdminClient().lookups().lookupTopic(heartbeatV2Topic), expectedBrokerServiceUrl); + } + + private void assertLookupSLANamespaceOwner(PulsarService pulsar, + String lookupServiceAddress, + String expectedBrokerServiceUrl) throws Exception { + NamespaceName slaMonitorNamespace = getSLAMonitorNamespace(lookupServiceAddress, pulsar.getConfiguration()); + String slaMonitorTopic = slaMonitorNamespace.getPersistentTopicName("test"); + String result = pulsar.getAdminClient().lookups().lookupTopic(slaMonitorTopic); + log.info("Topic {} Lookup result: {}", slaMonitorTopic, result); + assertNotNull(result); + assertEquals(result, expectedBrokerServiceUrl); + } + @Test public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Exception { var topBundlesLoadDataStorePrimary = @@ -1024,15 +1077,15 @@ public void testListTopic() throws Exception { admin.namespaces().deleteNamespace(namespace, true); } - @Test(timeOut = 30 * 1000) + @Test(timeOut = 30 * 1000, priority = -1) public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws Exception { NamespaceName heartbeatNamespacePulsar1V1 = - NamespaceService.getHeartbeatNamespace(pulsar1.getLookupServiceAddress(), pulsar1.getConfiguration()); + getHeartbeatNamespace(pulsar1.getLookupServiceAddress(), pulsar1.getConfiguration()); NamespaceName heartbeatNamespacePulsar1V2 = NamespaceService.getHeartbeatNamespaceV2(pulsar1.getLookupServiceAddress(), pulsar1.getConfiguration()); NamespaceName heartbeatNamespacePulsar2V1 = - NamespaceService.getHeartbeatNamespace(pulsar2.getLookupServiceAddress(), pulsar2.getConfiguration()); + getHeartbeatNamespace(pulsar2.getLookupServiceAddress(), pulsar2.getConfiguration()); NamespaceName heartbeatNamespacePulsar2V2 = NamespaceService.getHeartbeatNamespaceV2(pulsar2.getLookupServiceAddress(), pulsar2.getConfiguration()); @@ -1049,22 +1102,22 @@ public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws Exceptio Set ownedServiceUnitsByPulsar1 = primaryLoadManager.getOwnedServiceUnits(); log.info("Owned service units: {}", ownedServiceUnitsByPulsar1); // heartbeat namespace bundle will own by pulsar1 - assertEquals(ownedServiceUnitsByPulsar1.size(), 2); + assertEquals(ownedServiceUnitsByPulsar1.size(), 3); assertTrue(ownedServiceUnitsByPulsar1.contains(bundle1)); assertTrue(ownedServiceUnitsByPulsar1.contains(bundle2)); Set ownedServiceUnitsByPulsar2 = secondaryLoadManager.getOwnedServiceUnits(); log.info("Owned service units: {}", ownedServiceUnitsByPulsar2); - assertEquals(ownedServiceUnitsByPulsar2.size(), 2); + assertEquals(ownedServiceUnitsByPulsar2.size(), 3); assertTrue(ownedServiceUnitsByPulsar2.contains(bundle3)); assertTrue(ownedServiceUnitsByPulsar2.contains(bundle4)); Map ownedNamespacesByPulsar1 = admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar1.getLookupServiceAddress()); Map ownedNamespacesByPulsar2 = admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar2.getLookupServiceAddress()); - assertEquals(ownedNamespacesByPulsar1.size(), 2); + assertEquals(ownedNamespacesByPulsar1.size(), 3); assertTrue(ownedNamespacesByPulsar1.containsKey(bundle1.toString())); assertTrue(ownedNamespacesByPulsar1.containsKey(bundle2.toString())); - assertEquals(ownedNamespacesByPulsar2.size(), 2); + assertEquals(ownedNamespacesByPulsar2.size(), 3); assertTrue(ownedNamespacesByPulsar2.containsKey(bundle3.toString())); assertTrue(ownedNamespacesByPulsar2.containsKey(bundle4.toString())); @@ -1115,7 +1168,8 @@ public void testTryAcquiringOwnership() String topic = "persistent://" + namespace + "/test"; NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).get(); NamespaceEphemeralData namespaceEphemeralData = primaryLoadManager.tryAcquiringOwnership(bundle).get(); - assertEquals(namespaceEphemeralData.getNativeUrl(), pulsar1.getBrokerServiceUrl()); + assertTrue(Set.of(pulsar1.getBrokerServiceUrl(), pulsar2.getBrokerServiceUrl()) + .contains(namespaceEphemeralData.getNativeUrl())); admin.namespaces().deleteNamespace(namespace, true); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index caf7f0d5d5eb0..6b6e4b2184a21 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -30,8 +30,6 @@ import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.EventType.Unload; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.MAX_CLEAN_UP_DELAY_TIME_IN_SECS; import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData.state; -import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_FMT; -import static org.apache.pulsar.broker.namespace.NamespaceService.HEARTBEAT_NAMESPACE_FMT_V2; import static org.apache.pulsar.metadata.api.extended.SessionEvent.ConnectionLost; import static org.apache.pulsar.metadata.api.extended.SessionEvent.Reconnected; import static org.apache.pulsar.metadata.api.extended.SessionEvent.SessionLost; @@ -89,7 +87,6 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.TableViewImpl; -import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; @@ -639,7 +636,7 @@ public void splitAndRetryTest() throws Exception { var leader = channel1.isChannelOwnerAsync().get() ? channel1 : channel2; validateMonitorCounters(leader, 0, - 1, + 3, 0, 0, 0, @@ -756,34 +753,6 @@ public void handleBrokerDeletionEventTest() waitUntilNewOwner(channel1, bundle2, broker); waitUntilNewOwner(channel2, bundle2, broker); - // Register the broker-1 heartbeat namespace bundle. - String heartbeatNamespaceBroker1V1 = NamespaceName - .get(String.format(HEARTBEAT_NAMESPACE_FMT, conf.getClusterName(), broker)).toString(); - String heartbeatNamespaceBroker1V2 = NamespaceName - .get(String.format(HEARTBEAT_NAMESPACE_FMT_V2, broker)).toString(); - String heartbeatNamespaceBroker1V1Bundle = heartbeatNamespaceBroker1V1 + "/0x00000000_0xfffffff0"; - String heartbeatNamespaceBroker1V2Bundle = heartbeatNamespaceBroker1V2 + "/0x00000000_0xfffffff0"; - channel1.publishAssignEventAsync(heartbeatNamespaceBroker1V1Bundle, broker); - channel1.publishAssignEventAsync(heartbeatNamespaceBroker1V2Bundle, broker); - - // Register the broker-2 heartbeat namespace bundle. - String heartbeatNamespaceBroker2V1 = NamespaceName - .get(String.format(HEARTBEAT_NAMESPACE_FMT, conf.getClusterName(), lookupServiceAddress2)).toString(); - String heartbeatNamespaceBroker2V2 = NamespaceName - .get(String.format(HEARTBEAT_NAMESPACE_FMT_V2, lookupServiceAddress2)).toString(); - String heartbeatNamespaceBroker2V1Bundle = heartbeatNamespaceBroker2V1 + "/0x00000000_0xfffffff0"; - String heartbeatNamespaceBroker2V2Bundle = heartbeatNamespaceBroker2V2 + "/0x00000000_0xfffffff0"; - channel1.publishAssignEventAsync(heartbeatNamespaceBroker2V1Bundle, lookupServiceAddress2); - channel1.publishAssignEventAsync(heartbeatNamespaceBroker2V2Bundle, lookupServiceAddress2); - waitUntilNewOwner(channel1, heartbeatNamespaceBroker1V1Bundle, broker); - waitUntilNewOwner(channel1, heartbeatNamespaceBroker1V2Bundle, broker); - waitUntilNewOwner(channel2, heartbeatNamespaceBroker1V1Bundle, broker); - waitUntilNewOwner(channel2, heartbeatNamespaceBroker1V2Bundle, broker); - waitUntilNewOwner(channel1, heartbeatNamespaceBroker2V1Bundle, lookupServiceAddress2); - waitUntilNewOwner(channel1, heartbeatNamespaceBroker2V2Bundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, heartbeatNamespaceBroker2V1Bundle, lookupServiceAddress2); - waitUntilNewOwner(channel2, heartbeatNamespaceBroker2V2Bundle, lookupServiceAddress2); - // Verify to transfer the ownership to the other broker. channel1.publishUnloadEventAsync(new Unload(broker, bundle1, Optional.of(lookupServiceAddress2))); waitUntilNewOwner(channel1, bundle1, lookupServiceAddress2); @@ -806,16 +775,6 @@ public void handleBrokerDeletionEventTest() waitUntilNewOwner(channel1, bundle2, lookupServiceAddress2); waitUntilNewOwner(channel2, bundle2, lookupServiceAddress2); - waitUntilNewOwner(channel1, heartbeatNamespaceBroker1V1Bundle, null); - waitUntilNewOwner(channel1, heartbeatNamespaceBroker1V2Bundle, null); - waitUntilNewOwner(channel2, heartbeatNamespaceBroker1V1Bundle, null); - waitUntilNewOwner(channel2, heartbeatNamespaceBroker1V2Bundle, null); - - waitUntilNewOwner(channel1, heartbeatNamespaceBroker2V1Bundle, null); - waitUntilNewOwner(channel1, heartbeatNamespaceBroker2V2Bundle, null); - waitUntilNewOwner(channel2, heartbeatNamespaceBroker2V1Bundle, null); - waitUntilNewOwner(channel2, heartbeatNamespaceBroker2V2Bundle, null); - verify(leaderCleanupJobs, times(1)).computeIfAbsent(eq(broker), any()); verify(followerCleanupJobs, times(0)).computeIfAbsent(eq(broker), any()); @@ -827,7 +786,7 @@ public void handleBrokerDeletionEventTest() validateMonitorCounters(leaderChannel, 2, 0, - 7, + 3, 0, 2, 0, @@ -858,7 +817,7 @@ public void handleBrokerDeletionEventTest() validateMonitorCounters(leaderChannel, 2, 0, - 7, + 3, 0, 3, 0, @@ -879,7 +838,7 @@ public void handleBrokerDeletionEventTest() validateMonitorCounters(leaderChannel, 2, 0, - 7, + 3, 0, 3, 0, @@ -901,7 +860,7 @@ public void handleBrokerDeletionEventTest() validateMonitorCounters(leaderChannel, 2, 0, - 7, + 3, 0, 4, 0, @@ -923,7 +882,7 @@ public void handleBrokerDeletionEventTest() validateMonitorCounters(leaderChannel, 3, 0, - 9, + 5, 0, 4, 0, @@ -952,7 +911,7 @@ public void handleBrokerDeletionEventTest() validateMonitorCounters(leaderChannel, 3, 0, - 9, + 5, 0, 4, 1, @@ -1447,7 +1406,7 @@ public void splitAndRetryFailureTest() throws Exception { validateMonitorCounters(leader, 0, - 1, + 3, 1, 0, 0, From b765a02e6294203ac6619dbd73546f4f74a7627d Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Sun, 22 Oct 2023 22:52:19 -0500 Subject: [PATCH 323/494] [fix][test] Fix AuditorLedgerCheckerTest flaky test. (#21414) --- .../apache/bookkeeper/replication/AuditorLedgerCheckerTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index 12bc6fbe55d24..fcaadaa9bbedd 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -53,7 +53,6 @@ import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.proto.BookieServer; -import org.apache.bookkeeper.test.BookKeeperClusterTestCase; import org.apache.bookkeeper.client.LedgerMetadataBuilder; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.common.util.OrderedScheduler; From 480cb3bbc22c57f52a969a935aaf468bf37af351 Mon Sep 17 00:00:00 2001 From: Yan Zhao Date: Mon, 23 Oct 2023 09:20:35 -0500 Subject: [PATCH 324/494] [fix] [test] [branch-3.0] Fix AutoRecovery flaky test. (#21418) --- .../bookkeeper/PulsarLedgerAuditorManager.java | 13 +++++++++++++ .../replication/AuditorLedgerCheckerTest.java | 14 +++++++------- .../replication/AuditorPeriodicCheckTest.java | 8 ++++---- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java index 957f26ecd2e80..44870ed47f05b 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarLedgerAuditorManager.java @@ -27,6 +27,7 @@ import org.apache.pulsar.metadata.api.coordination.LeaderElection; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; +import org.apache.pulsar.metadata.api.extended.SessionEvent; import org.apache.pulsar.metadata.coordination.impl.CoordinationServiceImpl; @Slf4j @@ -38,6 +39,7 @@ public class PulsarLedgerAuditorManager implements LedgerAuditorManager { private final LeaderElection leaderElection; private LeaderElectionState leaderElectionState; private String bookieId; + private boolean sessionExpired = false; PulsarLedgerAuditorManager(MetadataStoreExtended store, String ledgersRoot) { this.coordinationService = new CoordinationServiceImpl(store); @@ -47,6 +49,14 @@ public class PulsarLedgerAuditorManager implements LedgerAuditorManager { this.leaderElection = coordinationService.getLeaderElection(String.class, electionPath, this::handleStateChanges); this.leaderElectionState = LeaderElectionState.NoLeader; + store.registerSessionListener(event -> { + if (SessionEvent.SessionLost == event) { + synchronized (this) { + sessionExpired = true; + notifyAll(); + } + } + }); } private void handleStateChanges(LeaderElectionState state) { @@ -71,6 +81,9 @@ public void tryToBecomeAuditor(String bookieId, Consumer listener) while (true) { try { synchronized (this) { + if (sessionExpired) { + throw new IllegalStateException("Zookeeper session expired, give up to become auditor."); + } if (leaderElectionState == LeaderElectionState.Leading) { return; } else { diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index fcaadaa9bbedd..ec5f77f79464b 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -23,10 +23,10 @@ import static org.testng.AssertJUnit.assertNotNull; import static org.testng.AssertJUnit.assertNotSame; import static org.testng.AssertJUnit.assertTrue; -import java.nio.ByteBuffer; -import java.util.ArrayList; import static org.testng.AssertJUnit.fail; import java.net.URI; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -40,19 +40,15 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import java.util.concurrent.TimeoutException; import lombok.Cleanup; import org.apache.bookkeeper.bookie.BookieImpl; import org.apache.bookkeeper.client.AsyncCallback.AddCallback; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerHandle; -import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; -import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; -import org.apache.bookkeeper.net.BookieId; -import org.apache.bookkeeper.proto.BookieServer; import org.apache.bookkeeper.client.LedgerMetadataBuilder; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.common.util.OrderedScheduler; @@ -64,7 +60,11 @@ import org.apache.bookkeeper.meta.MetadataClientDriver; import org.apache.bookkeeper.meta.MetadataDrivers; import org.apache.bookkeeper.meta.UnderreplicatedLedger; +import org.apache.bookkeeper.meta.ZkLedgerUnderreplicationManager; +import org.apache.bookkeeper.meta.zk.ZKMetadataDriverBase; +import org.apache.bookkeeper.net.BookieId; import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.proto.BookieServer; import org.apache.bookkeeper.replication.ReplicationException.UnavailableException; import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.pulsar.metadata.api.MetadataStoreConfig; diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java index 0fa6538b3ed43..9c5805dc536d6 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicCheckTest.java @@ -72,8 +72,8 @@ import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testng.annotations.AfterTest; -import org.testng.annotations.BeforeTest; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; /** @@ -96,7 +96,7 @@ public AuditorPeriodicCheckTest() throws Exception { Class.forName("org.apache.pulsar.metadata.bookkeeper.PulsarMetadataBookieDriver"); } - @BeforeTest + @BeforeMethod @Override public void setUp() throws Exception { super.setUp(); @@ -127,7 +127,7 @@ public void setUp() throws Exception { driver.initialize(serverConfiguration, NullStatsLogger.INSTANCE); } - @AfterTest + @AfterMethod @Override public void tearDown() throws Exception { if (null != driver) { From ad494eaf9b8b9c8e988d58d8c265e10a4a6b041e Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:22:44 -0700 Subject: [PATCH 325/494] [improve][broker] use ConcurrentHashMap in ServiceUnitStateChannel and avoid recursive update error (#21282) (cherry picked from commit aecdb03e0e64605d60f03d9b76f99c1136677dff) --- .../channel/ServiceUnitStateChannelImpl.java | 90 +++++++++++-------- .../channel/ServiceUnitStateChannelTest.java | 10 +-- 2 files changed, 59 insertions(+), 41 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index bb5b587219133..02acb923c2d94 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -54,6 +54,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledFuture; @@ -67,6 +68,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.lang3.mutable.MutableObject; import org.apache.pulsar.PulsarClusterMetadataSetup; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; @@ -97,7 +99,6 @@ import org.apache.pulsar.common.stats.Metrics; import org.apache.pulsar.common.topics.TopicCompactionStrategy; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.extended.SessionEvent; @@ -125,9 +126,9 @@ public class ServiceUnitStateChannelImpl implements ServiceUnitStateChannel { private final PulsarService pulsar; private final ServiceConfiguration config; private final Schema schema; - private final ConcurrentOpenHashMap> getOwnerRequests; + private final Map> getOwnerRequests; private final String lookupServiceAddress; - private final ConcurrentOpenHashMap> cleanupJobs; + private final Map> cleanupJobs; private final StateChangeListeners stateChangeListeners; private ExtensibleLoadManagerImpl loadManager; private BrokerRegistry brokerRegistry; @@ -204,9 +205,8 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.config = pulsar.getConfig(); this.lookupServiceAddress = pulsar.getLookupServiceAddress(); this.schema = Schema.JSON(ServiceUnitStateData.class); - this.getOwnerRequests = ConcurrentOpenHashMap.>newBuilder().build(); - this.cleanupJobs = ConcurrentOpenHashMap.>newBuilder().build(); + this.getOwnerRequests = new ConcurrentHashMap<>(); + this.cleanupJobs = new ConcurrentHashMap<>(); this.stateChangeListeners = new StateChangeListeners(); this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateTombstoneDelayTimeInSeconds() * 1000; @@ -826,20 +826,28 @@ private boolean isTargetBroker(String broker) { } private CompletableFuture deferGetOwnerRequest(String serviceUnit) { - return getOwnerRequests - .computeIfAbsent(serviceUnit, k -> { - CompletableFuture future = new CompletableFuture<>(); - future.orTimeout(inFlightStateWaitingTimeInMillis, TimeUnit.MILLISECONDS) - .whenComplete((v, e) -> { - if (e != null) { - getOwnerRequests.remove(serviceUnit, future); - log.warn("Failed to getOwner for serviceUnit:{}", - serviceUnit, e); - } + var requested = new MutableObject>(); + try { + return getOwnerRequests + .computeIfAbsent(serviceUnit, k -> { + CompletableFuture future = new CompletableFuture<>(); + requested.setValue(future); + return future; + }); + } finally { + var future = requested.getValue(); + if (future != null) { + future.orTimeout(inFlightStateWaitingTimeInMillis, TimeUnit.MILLISECONDS) + .whenComplete((v, e) -> { + if (e != null) { + getOwnerRequests.remove(serviceUnit, future); + log.warn("Failed to getOwner for serviceUnit:{}", + serviceUnit, e); } - ); - return future; - }); + } + ); + } + } } private CompletableFuture closeServiceUnit(String serviceUnit) { @@ -1113,24 +1121,34 @@ private void handleBrokerDeletionEvent(String broker) { } private void scheduleCleanup(String broker, long delayInSecs) { - cleanupJobs.computeIfAbsent(broker, k -> { - Executor delayed = CompletableFuture - .delayedExecutor(delayInSecs, TimeUnit.SECONDS, pulsar.getLoadManagerExecutor()); - totalInactiveBrokerCleanupScheduledCnt++; - return CompletableFuture - .runAsync(() -> { - try { - doCleanup(broker); - } catch (Throwable e) { - log.error("Failed to run the cleanup job for the broker {}, " - + "totalCleanupErrorCnt:{}.", - broker, totalCleanupErrorCnt.incrementAndGet(), e); - } finally { - cleanupJobs.remove(broker); + var scheduled = new MutableObject>(); + try { + cleanupJobs.computeIfAbsent(broker, k -> { + Executor delayed = CompletableFuture + .delayedExecutor(delayInSecs, TimeUnit.SECONDS, pulsar.getLoadManagerExecutor()); + totalInactiveBrokerCleanupScheduledCnt++; + var future = CompletableFuture + .runAsync(() -> { + try { + doCleanup(broker); + } catch (Throwable e) { + log.error("Failed to run the cleanup job for the broker {}, " + + "totalCleanupErrorCnt:{}.", + broker, totalCleanupErrorCnt.incrementAndGet(), e); + } } - } - , delayed); - }); + , delayed); + scheduled.setValue(future); + return future; + }); + } finally { + var future = scheduled.getValue(); + if (future != null) { + future.whenComplete((v, ex) -> { + cleanupJobs.remove(broker); + }); + } + } log.info("Scheduled ownership cleanup for broker:{} with delay:{} secs. Pending clean jobs:{}.", broker, delayInSecs, cleanupJobs.size()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 6b6e4b2184a21..990408d214b7f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -60,6 +60,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -88,7 +89,6 @@ import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.policies.data.TopicType; -import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.NotificationType; import org.apache.pulsar.metadata.api.coordination.LeaderElectionState; @@ -1558,9 +1558,9 @@ public void testOverrideOrphanStateData() } - private static ConcurrentOpenHashMap>> getOwnerRequests( + private static ConcurrentHashMap>> getOwnerRequests( ServiceUnitStateChannel channel) throws IllegalAccessException { - return (ConcurrentOpenHashMap>>) + return (ConcurrentHashMap>>) FieldUtils.readDeclaredField(channel, "getOwnerRequests", true); } @@ -1577,9 +1577,9 @@ private static long getLastMetadataSessionEventTimestamp(ServiceUnitStateChannel FieldUtils.readField(channel, "lastMetadataSessionEventTimestamp", true); } - private static ConcurrentOpenHashMap> getCleanupJobs( + private static ConcurrentHashMap> getCleanupJobs( ServiceUnitStateChannel channel) throws IllegalAccessException { - return (ConcurrentOpenHashMap>) + return (ConcurrentHashMap>) FieldUtils.readField(channel, "cleanupJobs", true); } From 5503071a6b1d821a6abe026c9a98694592f84a16 Mon Sep 17 00:00:00 2001 From: ken <1647023764@qq.com> Date: Thu, 19 Oct 2023 10:18:12 +0800 Subject: [PATCH 326/494] [fix][broker] Fix heartbeat namespace create transaction internal topic (#21348) (cherry picked from commit c8a2f49c6c6edaf6b5667f9ac7df65b815aefe58) --- .../service/persistent/PersistentTopic.java | 3 ++- .../systopic/PartitionedSystemTopicTest.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 22459b2f61575..aa1b344558648 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -326,7 +326,8 @@ public PersistentTopic(String topic, ManagedLedger ledger, BrokerService brokerS checkReplicatedSubscriptionControllerState(); TopicName topicName = TopicName.get(topic); if (brokerService.getPulsar().getConfiguration().isTransactionCoordinatorEnabled() - && !isEventSystemTopic(topicName)) { + && !isEventSystemTopic(topicName) + && !NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject())) { this.transactionBuffer = brokerService.getPulsar() .getTransactionBufferProvider().newTransactionBuffer(this); } else { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index 4af0bd9052391..42d941e616809 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -81,6 +81,7 @@ protected void setup() throws Exception { conf.setDefaultNumPartitions(PARTITIONS); conf.setManagedLedgerMaxEntriesPerLedger(1); conf.setBrokerDeleteInactiveTopicsEnabled(false); + conf.setTransactionCoordinatorEnabled(true); super.baseSetup(); } @@ -207,6 +208,24 @@ public void testHeartbeatTopicNotAllowedToSendEvent() throws Exception { }); } + @Test + public void testHeartbeatNamespaceNotCreateTransactionInternalTopic() throws Exception { + admin.brokers().healthcheck(TopicVersion.V2); + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), + pulsar.getConfig()); + TopicName topicName = TopicName.get("persistent", + namespaceName, SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + Optional optionalTopic = pulsar.getBrokerService() + .getTopic(topicName.getPartition(1).toString(), false).join(); + Assert.assertTrue(optionalTopic.isEmpty()); + + List topics = getPulsar().getNamespaceService().getListOfPersistentTopics(namespaceName).join(); + Assert.assertEquals(topics.size(), 1); + TopicName heartbeatTopicName = TopicName.get("persistent", + namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); + Assert.assertEquals(topics.get(0), heartbeatTopicName.toString()); + } + @Test public void testSetBacklogCausedCreatingProducerFailure() throws Exception { final String ns = "prop/ns-test"; From 2290073d5d61cfac61d7fdff544992089fd22da5 Mon Sep 17 00:00:00 2001 From: ken <1647023764@qq.com> Date: Thu, 19 Oct 2023 18:39:41 +0800 Subject: [PATCH 327/494] [fix][broker] Fix heartbeat namespace create event topic and cannot delete heartbeat topic (#21360) Co-authored-by: fanjianye Co-authored-by: Jiwei Guo (cherry picked from commit 700a29d5c877dcde5f3c8c1e946b00a8296b8d4f) --- .../SystemTopicBasedTopicPoliciesService.java | 14 +++++++---- .../systopic/PartitionedSystemTopicTest.java | 23 +++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 09f8de818db0a..9651e76a25e93 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -92,20 +92,23 @@ public SystemTopicBasedTopicPoliciesService(PulsarService pulsarService) { @Override public CompletableFuture deleteTopicPoliciesAsync(TopicName topicName) { + if (NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject())) { + return CompletableFuture.completedFuture(null); + } return sendTopicPolicyEvent(topicName, ActionType.DELETE, null); } @Override public CompletableFuture updateTopicPoliciesAsync(TopicName topicName, TopicPolicies policies) { + if (NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject())) { + return CompletableFuture.failedFuture(new BrokerServiceException.NotAllowedException( + "Not allowed to update topic policy for the heartbeat topic")); + } return sendTopicPolicyEvent(topicName, ActionType.UPDATE, policies); } private CompletableFuture sendTopicPolicyEvent(TopicName topicName, ActionType actionType, TopicPolicies policies) { - if (NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject())) { - return CompletableFuture.failedFuture( - new BrokerServiceException.NotAllowedException("Not allowed to send event to health check topic")); - } return pulsarService.getPulsarResources().getNamespaceResources() .getPoliciesAsync(topicName.getNamespaceObject()) .thenCompose(namespacePolicies -> { @@ -217,6 +220,9 @@ public TopicPolicies getTopicPolicies(TopicName topicName) throws TopicPoliciesC @Override public TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) throws TopicPoliciesCacheNotInitException { + if (NamespaceService.isHeartbeatNamespace(topicName.getNamespaceObject())) { + return null; + } if (!policyCacheInitMap.containsKey(topicName.getNamespaceObject())) { NamespaceName namespace = topicName.getNamespaceObject(); prepareInitPoliciesCache(namespace, new CompletableFuture<>()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java index 42d941e616809..416d7ed02708e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/PartitionedSystemTopicTest.java @@ -191,6 +191,13 @@ public void testSystemNamespaceNotCreateChangeEventsTopic() throws Exception { Optional optionalTopic = pulsar.getBrokerService() .getTopic(topicName.getPartition(1).toString(), false).join(); Assert.assertTrue(optionalTopic.isEmpty()); + + TopicName heartbeatTopicName = TopicName.get("persistent", + namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); + admin.topics().getRetention(heartbeatTopicName.toString()); + optionalTopic = pulsar.getBrokerService() + .getTopic(topicName.getPartition(1).toString(), false).join(); + Assert.assertTrue(optionalTopic.isEmpty()); } @Test @@ -208,6 +215,22 @@ public void testHeartbeatTopicNotAllowedToSendEvent() throws Exception { }); } + @Test + public void testHeartbeatTopicBeDeleted() throws Exception { + admin.brokers().healthcheck(TopicVersion.V2); + NamespaceName namespaceName = NamespaceService.getHeartbeatNamespaceV2(pulsar.getLookupServiceAddress(), + pulsar.getConfig()); + TopicName heartbeatTopicName = TopicName.get("persistent", namespaceName, BrokersBase.HEALTH_CHECK_TOPIC_SUFFIX); + + List topics = getPulsar().getNamespaceService().getListOfPersistentTopics(namespaceName).join(); + Assert.assertEquals(topics.size(), 1); + Assert.assertEquals(topics.get(0), heartbeatTopicName.toString()); + + admin.topics().delete(heartbeatTopicName.toString(), true); + topics = getPulsar().getNamespaceService().getListOfPersistentTopics(namespaceName).join(); + Assert.assertEquals(topics.size(), 0); + } + @Test public void testHeartbeatNamespaceNotCreateTransactionInternalTopic() throws Exception { admin.brokers().healthcheck(TopicVersion.V2); From cefdcd8dc195573dc8f00b17886d81b8ada54aa2 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 23 Oct 2023 15:05:18 +0300 Subject: [PATCH 328/494] [fix][test] Fix LocalBookkeeperEnsemble resource leak in tests (#21407) (cherry picked from commit a0f8b0d16840118388f258a6ef7ac6e76d564a15) --- .../zookeeper/LocalBookkeeperEnsemble.java | 10 ++++++---- .../LocalBookkeeperEnsembleTest.java | 20 ------------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java b/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java index d73d1d7ed6bed..63d146a3a1521 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java @@ -52,7 +52,6 @@ import org.apache.bookkeeper.clients.exceptions.NamespaceExistsException; import org.apache.bookkeeper.clients.exceptions.NamespaceNotFoundException; import org.apache.bookkeeper.common.allocator.PoolingPolicy; -import org.apache.bookkeeper.common.component.ComponentStarter; import org.apache.bookkeeper.common.component.LifecycleComponent; import org.apache.bookkeeper.common.component.LifecycleComponentStack; import org.apache.bookkeeper.common.concurrent.FutureUtils; @@ -132,7 +131,7 @@ public LocalBookkeeperEnsemble(int numberOfBookies, boolean clearOldData, String advertisedAddress) { this(numberOfBookies, zkPort, streamStoragePort, zkDataDirName, bkDataDirName, clearOldData, advertisedAddress, - new BasePortManager(bkBasePort)); + bkBasePort != 0 ? new BasePortManager(bkBasePort) : () -> 0); } public LocalBookkeeperEnsemble(int numberOfBookies, @@ -311,6 +310,7 @@ private void runBookies(ServerConfiguration baseConf) throws Exception { bsConfs[i] = new ServerConfiguration(baseConf); // override settings bsConfs[i].setBookiePort(bookiePort); + bsConfs[i].setBookieId("bk" + i + "test"); String zkServers = "127.0.0.1:" + zkPort; String metadataServiceUriStr = "zk://" + zkServers + "/ledgers"; @@ -455,8 +455,10 @@ public void startBK(int i) throws Exception { try { bookieComponents[i] = org.apache.bookkeeper.server.Main .buildBookieServer(new BookieConfiguration(bsConfs[i])); - ComponentStarter.startComponent(bookieComponents[i]); + bookieComponents[i].start(); } catch (BookieException.InvalidCookieException ice) { + LOG.warn("Invalid cookie found for bookie {}", i, ice); + // InvalidCookieException can happen if the machine IP has changed // Since we are running here a local bookie that is always accessed // from localhost, we can ignore the error @@ -473,7 +475,7 @@ public void startBK(int i) throws Exception { bookieComponents[i] = org.apache.bookkeeper.server.Main .buildBookieServer(new BookieConfiguration(bsConfs[i])); - ComponentStarter.startComponent(bookieComponents[i]); + bookieComponents[i].start(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsembleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsembleTest.java index 92899feda7371..a4bc69a7266cc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsembleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsembleTest.java @@ -21,10 +21,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; - -import java.util.Collections; -import java.util.List; - import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -39,22 +35,6 @@ void setup() throws Exception { void teardown() throws Exception { } - @Test - public void testAdvertisedAddress() throws Exception { - final int numBk = 1; - - LocalBookkeeperEnsemble ensemble = new LocalBookkeeperEnsemble( - numBk, 0, 0, null, null, true, "127.0.0.2"); - ensemble.startStandalone(); - - List bookies = ensemble.getZkClient().getChildren("/ledgers/available", false); - Collections.sort(bookies); - assertEquals(bookies.size(), 2); - assertTrue(bookies.get(0).startsWith("127.0.0.2:")); - - ensemble.stop(); - } - @Test public void testStartStop() throws Exception { From a34bd59efce321dc32bd4d12843cb6e87889f539 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 19 Oct 2023 16:04:49 +0300 Subject: [PATCH 329/494] [fix][sec] Upgrade Jetty to 9.4.53 to address CVE-2023-44487 (#21395) (cherry picked from commit 22fd8c26c97238348b251980407ec3c338834f29) --- .../server/src/assemble/LICENSE.bin.txt | 38 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 16 ++++---- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 32 ++++++++-------- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index f622e89a68882..7e22e7dd2ae97 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -383,25 +383,25 @@ The Apache Software License, Version 2.0 - org.asynchttpclient-async-http-client-2.12.1.jar - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar * Jetty - - org.eclipse.jetty-jetty-client-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-continuation-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-http-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-io-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-proxy-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-security-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-server-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-servlet-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-servlets-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-util-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-util-ajax-9.4.51.v20230217.jar - - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.51.v20230217.jar - - org.eclipse.jetty.websocket-websocket-api-9.4.51.v20230217.jar - - org.eclipse.jetty.websocket-websocket-client-9.4.51.v20230217.jar - - org.eclipse.jetty.websocket-websocket-common-9.4.51.v20230217.jar - - org.eclipse.jetty.websocket-websocket-server-9.4.51.v20230217.jar - - org.eclipse.jetty.websocket-websocket-servlet-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.51.v20230217.jar - - org.eclipse.jetty-jetty-alpn-server-9.4.51.v20230217.jar + - org.eclipse.jetty-jetty-client-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-continuation-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-proxy-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-servlets-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar + - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.53.v20231009.jar + - org.eclipse.jetty.websocket-websocket-api-9.4.53.v20231009.jar + - org.eclipse.jetty.websocket-websocket-client-9.4.53.v20231009.jar + - org.eclipse.jetty.websocket-websocket-common-9.4.53.v20231009.jar + - org.eclipse.jetty.websocket-websocket-server-9.4.53.v20231009.jar + - org.eclipse.jetty.websocket-websocket-servlet-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.53.v20231009.jar + - org.eclipse.jetty-jetty-alpn-server-9.4.53.v20231009.jar * SnakeYaml -- org.yaml-snakeyaml-2.0.jar * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 66ee8dbb4519e..0a962085603f2 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -399,14 +399,14 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Jetty - - jetty-client-9.4.51.v20230217.jar - - jetty-http-9.4.51.v20230217.jar - - jetty-io-9.4.51.v20230217.jar - - jetty-util-9.4.51.v20230217.jar - - javax-websocket-client-impl-9.4.51.v20230217.jar - - websocket-api-9.4.51.v20230217.jar - - websocket-client-9.4.51.v20230217.jar - - websocket-common-9.4.51.v20230217.jar + - jetty-client-9.4.53.v20231009.jar + - jetty-http-9.4.53.v20231009.jar + - jetty-io-9.4.53.v20231009.jar + - jetty-util-9.4.53.v20231009.jar + - javax-websocket-client-impl-9.4.53.v20231009.jar + - websocket-api-9.4.53.v20231009.jar + - websocket-client-9.4.53.v20231009.jar + - websocket-common-9.4.53.v20231009.jar * SnakeYaml -- snakeyaml-2.0.jar * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar * Javassist -- javassist-3.25.0-GA.jar diff --git a/pom.xml b/pom.xml index 5b8170cec3722..8ef8714bd0132 100644 --- a/pom.xml +++ b/pom.xml @@ -142,7 +142,7 @@ flexible messaging model and an intuitive client API. 5.1.0 4.1.94.Final 0.0.21.Final - 9.4.51.v20230217 + 9.4.53.v20231009 2.5.2 2.34 1.10.50 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index ea4b3fe93b89f..09ed6a3605904 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -279,22 +279,22 @@ The Apache Software License, Version 2.0 - joda-time-2.10.10.jar - failsafe-2.4.4.jar * Jetty - - http2-client-9.4.51.v20230217.jar - - http2-common-9.4.51.v20230217.jar - - http2-hpack-9.4.51.v20230217.jar - - http2-http-client-transport-9.4.51.v20230217.jar - - jetty-alpn-client-9.4.51.v20230217.jar - - http2-server-9.4.51.v20230217.jar - - jetty-alpn-java-client-9.4.51.v20230217.jar - - jetty-client-9.4.51.v20230217.jar - - jetty-http-9.4.51.v20230217.jar - - jetty-io-9.4.51.v20230217.jar - - jetty-jmx-9.4.51.v20230217.jar - - jetty-security-9.4.51.v20230217.jar - - jetty-server-9.4.51.v20230217.jar - - jetty-servlet-9.4.51.v20230217.jar - - jetty-util-9.4.51.v20230217.jar - - jetty-util-ajax-9.4.51.v20230217.jar + - http2-client-9.4.53.v20231009.jar + - http2-common-9.4.53.v20231009.jar + - http2-hpack-9.4.53.v20231009.jar + - http2-http-client-transport-9.4.53.v20231009.jar + - jetty-alpn-client-9.4.53.v20231009.jar + - http2-server-9.4.53.v20231009.jar + - jetty-alpn-java-client-9.4.53.v20231009.jar + - jetty-client-9.4.53.v20231009.jar + - jetty-http-9.4.53.v20231009.jar + - jetty-io-9.4.53.v20231009.jar + - jetty-jmx-9.4.53.v20231009.jar + - jetty-security-9.4.53.v20231009.jar + - jetty-server-9.4.53.v20231009.jar + - jetty-servlet-9.4.53.v20231009.jar + - jetty-util-9.4.53.v20231009.jar + - jetty-util-ajax-9.4.53.v20231009.jar * Byte Buddy - byte-buddy-1.11.13.jar * Apache BVal From b931ad23a2981a4c9ebb6ff2d6d4edd98928ac28 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 19 Oct 2023 16:05:09 +0300 Subject: [PATCH 330/494] [fix][sec] Upgrade Netty to 4.1.100 to address CVE-2023-44487 (#21397) (cherry picked from commit aae6c716b6f7b32c96484b9004b62359e27f158e) --- buildtools/pom.xml | 2 +- .../server/src/assemble/LICENSE.bin.txt | 42 +++++++++---------- .../shell/src/assemble/LICENSE.bin.txt | 38 ++++++++--------- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 40 +++++++++--------- 5 files changed, 62 insertions(+), 62 deletions(-) diff --git a/buildtools/pom.xml b/buildtools/pom.xml index b8595959853f0..9fd1db11a503c 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -47,7 +47,7 @@ 4.1 8.37 3.1.2 - 4.1.94.Final + 4.1.100.Final 4.2.3 32.1.1-jre 1.10.12 diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 7e22e7dd2ae97..eafc4000700b8 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -289,27 +289,27 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-lang3-3.11.jar - org.apache.commons-commons-text-1.10.0.jar * Netty - - io.netty-netty-buffer-4.1.94.Final.jar - - io.netty-netty-codec-4.1.94.Final.jar - - io.netty-netty-codec-dns-4.1.94.Final.jar - - io.netty-netty-codec-http-4.1.94.Final.jar - - io.netty-netty-codec-http2-4.1.94.Final.jar - - io.netty-netty-codec-socks-4.1.94.Final.jar - - io.netty-netty-codec-haproxy-4.1.94.Final.jar - - io.netty-netty-common-4.1.94.Final.jar - - io.netty-netty-handler-4.1.94.Final.jar - - io.netty-netty-handler-proxy-4.1.94.Final.jar - - io.netty-netty-resolver-4.1.94.Final.jar - - io.netty-netty-resolver-dns-4.1.94.Final.jar - - io.netty-netty-resolver-dns-classes-macos-4.1.94.Final.jar - - io.netty-netty-resolver-dns-native-macos-4.1.94.Final-osx-aarch_64.jar - - io.netty-netty-resolver-dns-native-macos-4.1.94.Final-osx-x86_64.jar - - io.netty-netty-transport-4.1.94.Final.jar - - io.netty-netty-transport-classes-epoll-4.1.94.Final.jar - - io.netty-netty-transport-native-epoll-4.1.94.Final-linux-x86_64.jar - - io.netty-netty-transport-native-epoll-4.1.94.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.94.Final.jar - - io.netty-netty-transport-native-unix-common-4.1.94.Final-linux-x86_64.jar + - io.netty-netty-buffer-4.1.100.Final.jar + - io.netty-netty-codec-4.1.100.Final.jar + - io.netty-netty-codec-dns-4.1.100.Final.jar + - io.netty-netty-codec-http-4.1.100.Final.jar + - io.netty-netty-codec-http2-4.1.100.Final.jar + - io.netty-netty-codec-socks-4.1.100.Final.jar + - io.netty-netty-codec-haproxy-4.1.100.Final.jar + - io.netty-netty-common-4.1.100.Final.jar + - io.netty-netty-handler-4.1.100.Final.jar + - io.netty-netty-handler-proxy-4.1.100.Final.jar + - io.netty-netty-resolver-4.1.100.Final.jar + - io.netty-netty-resolver-dns-4.1.100.Final.jar + - io.netty-netty-resolver-dns-classes-macos-4.1.100.Final.jar + - io.netty-netty-resolver-dns-native-macos-4.1.100.Final-osx-aarch_64.jar + - io.netty-netty-resolver-dns-native-macos-4.1.100.Final-osx-x86_64.jar + - io.netty-netty-transport-4.1.100.Final.jar + - io.netty-netty-transport-classes-epoll-4.1.100.Final.jar + - io.netty-netty-transport-native-epoll-4.1.100.Final-linux-x86_64.jar + - io.netty-netty-transport-native-epoll-4.1.100.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.100.Final.jar + - io.netty-netty-transport-native-unix-common-4.1.100.Final-linux-x86_64.jar - io.netty-netty-tcnative-boringssl-static-2.0.61.Final.jar - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 0a962085603f2..6fc5442cf7a18 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -348,22 +348,22 @@ The Apache Software License, Version 2.0 - commons-text-1.10.0.jar - commons-compress-1.21.jar * Netty - - netty-buffer-4.1.94.Final.jar - - netty-codec-4.1.94.Final.jar - - netty-codec-dns-4.1.94.Final.jar - - netty-codec-http-4.1.94.Final.jar - - netty-codec-socks-4.1.94.Final.jar - - netty-codec-haproxy-4.1.94.Final.jar - - netty-common-4.1.94.Final.jar - - netty-handler-4.1.94.Final.jar - - netty-handler-proxy-4.1.94.Final.jar - - netty-resolver-4.1.94.Final.jar - - netty-resolver-dns-4.1.94.Final.jar - - netty-transport-4.1.94.Final.jar - - netty-transport-classes-epoll-4.1.94.Final.jar - - netty-transport-native-epoll-4.1.94.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.94.Final.jar - - netty-transport-native-unix-common-4.1.94.Final-linux-x86_64.jar + - netty-buffer-4.1.100.Final.jar + - netty-codec-4.1.100.Final.jar + - netty-codec-dns-4.1.100.Final.jar + - netty-codec-http-4.1.100.Final.jar + - netty-codec-socks-4.1.100.Final.jar + - netty-codec-haproxy-4.1.100.Final.jar + - netty-common-4.1.100.Final.jar + - netty-handler-4.1.100.Final.jar + - netty-handler-proxy-4.1.100.Final.jar + - netty-resolver-4.1.100.Final.jar + - netty-resolver-dns-4.1.100.Final.jar + - netty-transport-4.1.100.Final.jar + - netty-transport-classes-epoll-4.1.100.Final.jar + - netty-transport-native-epoll-4.1.100.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.100.Final.jar + - netty-transport-native-unix-common-4.1.100.Final-linux-x86_64.jar - netty-tcnative-boringssl-static-2.0.61.Final.jar - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar @@ -374,9 +374,9 @@ The Apache Software License, Version 2.0 - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar - - netty-resolver-dns-classes-macos-4.1.94.Final.jar - - netty-resolver-dns-native-macos-4.1.94.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.94.Final-osx-x86_64.jar + - netty-resolver-dns-classes-macos-4.1.100.Final.jar + - netty-resolver-dns-native-macos-4.1.100.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.100.Final-osx-x86_64.jar * Prometheus client - simpleclient-0.16.0.jar - simpleclient_log4j2-0.16.0.jar diff --git a/pom.xml b/pom.xml index 8ef8714bd0132..c4fb95e8de24e 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ flexible messaging model and an intuitive client API. 1.1.10.1 4.1.12.1 5.1.0 - 4.1.94.Final + 4.1.100.Final 0.0.21.Final 9.4.53.v20231009 2.5.2 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 09ed6a3605904..6f0848ce4582f 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -231,21 +231,21 @@ The Apache Software License, Version 2.0 - commons-compress-1.21.jar - commons-lang3-3.11.jar * Netty - - netty-buffer-4.1.94.Final.jar - - netty-codec-4.1.94.Final.jar - - netty-codec-dns-4.1.94.Final.jar - - netty-codec-http-4.1.94.Final.jar - - netty-codec-haproxy-4.1.94.Final.jar - - netty-codec-socks-4.1.94.Final.jar - - netty-handler-proxy-4.1.94.Final.jar - - netty-common-4.1.94.Final.jar - - netty-handler-4.1.94.Final.jar + - netty-buffer-4.1.100.Final.jar + - netty-codec-4.1.100.Final.jar + - netty-codec-dns-4.1.100.Final.jar + - netty-codec-http-4.1.100.Final.jar + - netty-codec-haproxy-4.1.100.Final.jar + - netty-codec-socks-4.1.100.Final.jar + - netty-handler-proxy-4.1.100.Final.jar + - netty-common-4.1.100.Final.jar + - netty-handler-4.1.100.Final.jar - netty-reactive-streams-2.0.6.jar - - netty-resolver-4.1.94.Final.jar - - netty-resolver-dns-4.1.94.Final.jar - - netty-resolver-dns-classes-macos-4.1.94.Final.jar - - netty-resolver-dns-native-macos-4.1.94.Final-osx-aarch_64.jar - - netty-resolver-dns-native-macos-4.1.94.Final-osx-x86_64.jar + - netty-resolver-4.1.100.Final.jar + - netty-resolver-dns-4.1.100.Final.jar + - netty-resolver-dns-classes-macos-4.1.100.Final.jar + - netty-resolver-dns-native-macos-4.1.100.Final-osx-aarch_64.jar + - netty-resolver-dns-native-macos-4.1.100.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.61.Final.jar - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar @@ -253,12 +253,12 @@ The Apache Software License, Version 2.0 - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar - netty-tcnative-classes-2.0.61.Final.jar - - netty-transport-4.1.94.Final.jar - - netty-transport-classes-epoll-4.1.94.Final.jar - - netty-transport-native-epoll-4.1.94.Final-linux-x86_64.jar - - netty-transport-native-unix-common-4.1.94.Final.jar - - netty-transport-native-unix-common-4.1.94.Final-linux-x86_64.jar - - netty-codec-http2-4.1.94.Final.jar + - netty-transport-4.1.100.Final.jar + - netty-transport-classes-epoll-4.1.100.Final.jar + - netty-transport-native-epoll-4.1.100.Final-linux-x86_64.jar + - netty-transport-native-unix-common-4.1.100.Final.jar + - netty-transport-native-unix-common-4.1.100.Final-linux-x86_64.jar + - netty-codec-http2-4.1.100.Final.jar - netty-incubator-transport-classes-io_uring-0.0.21.Final.jar - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-x86_64.jar - netty-incubator-transport-native-io_uring-0.0.21.Final-linux-aarch_64.jar From 6d4a883e0f350b931d40435f95cf8aafb09a6317 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 19 Oct 2023 16:48:37 +0300 Subject: [PATCH 331/494] [fix][sec] Upgrade Zookeeper to 3.8.3 to address CVE-2023-44981 (#21398) (cherry picked from commit e5120ec68907525177f5add5c95b022f3106da1a) --- distribution/server/src/assemble/LICENSE.bin.txt | 6 +++--- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index eafc4000700b8..cdea68ac69049 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -480,9 +480,9 @@ The Apache Software License, Version 2.0 - io.vertx-vertx-web-4.3.8.jar - io.vertx-vertx-web-common-4.3.8.jar * Apache ZooKeeper - - org.apache.zookeeper-zookeeper-3.8.1.jar - - org.apache.zookeeper-zookeeper-jute-3.8.1.jar - - org.apache.zookeeper-zookeeper-prometheus-metrics-3.8.1.jar + - org.apache.zookeeper-zookeeper-3.8.3.jar + - org.apache.zookeeper-zookeeper-jute-3.8.3.jar + - org.apache.zookeeper-zookeeper-prometheus-metrics-3.8.3.jar * Snappy Java - org.xerial.snappy-snappy-java-1.1.10.1.jar * Google HTTP Client diff --git a/pom.xml b/pom.xml index c4fb95e8de24e..efb0e8570ee54 100644 --- a/pom.xml +++ b/pom.xml @@ -134,7 +134,7 @@ flexible messaging model and an intuitive client API. 1.21 4.16.3 - 3.8.1 + 3.8.3 1.5.0 1.10.0 1.1.10.1 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 6f0848ce4582f..ca117edc5341c 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -468,8 +468,8 @@ The Apache Software License, Version 2.0 - memory-0.8.3.jar - sketches-core-0.8.3.jar * Apache Zookeeper - - zookeeper-3.8.1.jar - - zookeeper-jute-3.8.1.jar + - zookeeper-3.8.3.jar + - zookeeper-jute-3.8.3.jar * Apache Yetus Audience Annotations - audience-annotations-0.12.0.jar * Swagger From 7edfb4f9f9fe50e4ade4db90eabf723506b79819 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 24 Oct 2023 20:59:54 +0800 Subject: [PATCH 332/494] [fix][broker][branch-3.0] Fix inconsistent topic policy (#21256) --- .../ProxySaslAuthenticationTest.java | 13 +- .../authentication/SaslAuthenticateTest.java | 11 +- .../pulsar/broker/service/BrokerService.java | 295 +++++++++--------- .../SystemTopicBasedTopicPoliciesService.java | 100 ++++-- .../broker/service/TopicPoliciesService.java | 40 +++ .../broker/admin/PersistentTopicsTest.java | 1 + .../broker/admin/TopicPoliciesTest.java | 2 +- .../TopicPoliciesWithBrokerRestartTest.java | 104 ++++++ .../pulsar/broker/admin/TopicsAuthTest.java | 2 + .../pulsar/broker/auth/AuthLogsTest.java | 2 + .../broker/auth/MockAuthentication.java | 6 +- .../service/BrokerBookieIsolationTest.java | 3 +- ...temTopicBasedTopicPoliciesServiceTest.java | 6 +- .../persistent/PersistentTopicTest.java | 3 +- ...enticationTlsHostnameVerificationTest.java | 6 + .../AuthorizationProducerConsumerTest.java | 5 + .../client/api/MutualAuthenticationTest.java | 2 +- ...okenAuthenticatedProducerConsumerTest.java | 3 + ...uth2AuthenticatedProducerConsumerTest.java | 14 +- ...reTlsProducerConsumerTestWithAuthTest.java | 22 ++ .../PatternTopicsConsumerImplAuthTest.java | 1 + .../standalone_no_client_auth.conf | 3 +- .../proxy/server/ProxyAuthenticationTest.java | 2 +- .../server/ProxyForwardAuthDataTest.java | 2 +- .../server/ProxyRolesEnforcementTest.java | 2 +- .../server/ProxyWithAuthorizationNegTest.java | 2 + .../ProxyWithoutServiceDiscoveryTest.java | 3 + .../pulsar/sql/presto/TestPulsarAuth.java | 4 + .../ExtensibleLoadManagerTest.java | 1 + .../integration/presto/TestPulsarSQLAuth.java | 1 + 30 files changed, 464 insertions(+), 197 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java index 261efe680f862..f0e45aa734afb 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/ProxySaslAuthenticationTest.java @@ -49,6 +49,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.auth.AuthenticationSasl; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.proxy.server.ProxyConfiguration; import org.apache.pulsar.proxy.server.ProxyService; import org.slf4j.Logger; @@ -193,15 +194,17 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setClusterName("test"); conf.setSuperUserRoles(ImmutableSet.of("client/" + localHostname + "@" + kdc.getRealm())); - - super.init(); - - lookupUrl = new URI(pulsar.getBrokerServiceUrl()); - // set admin auth, to verify admin web resources Map clientSaslConfig = new HashMap<>(); clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); clientSaslConfig.put("serverType", "broker"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(clientSaslConfig)); + + super.init(); + + lookupUrl = new URI(pulsar.getBrokerServiceUrl()); log.info("set client jaas section name: PulsarClient"); admin = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl.toString()) diff --git a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java index 8a0d0392d1333..76c6f023d9a36 100644 --- a/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java +++ b/pulsar-broker-auth-sasl/src/test/java/org/apache/pulsar/broker/authentication/SaslAuthenticateTest.java @@ -53,6 +53,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationSasl; import org.apache.pulsar.common.api.AuthData; import org.apache.pulsar.common.sasl.SaslConstants; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; @@ -180,7 +181,12 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setClusterName("test"); conf.setSuperUserRoles(ImmutableSet.of("client" + "@" + kdc.getRealm())); - + Map clientSaslConfig = new HashMap<>(); + clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); + clientSaslConfig.put("serverType", "broker"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationSasl.class.getName()); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(clientSaslConfig)); super.init(); lookupUrl = new URI(pulsar.getWebServiceAddress()); @@ -191,9 +197,6 @@ protected void setup() throws Exception { .authentication(authSasl)); // set admin auth, to verify admin web resources - Map clientSaslConfig = new HashMap<>(); - clientSaslConfig.put("saslJaasClientSectionName", "PulsarClient"); - clientSaslConfig.put("serverType", "broker"); log.info("set client jaas section name: PulsarClient"); admin = PulsarAdmin.builder() .serviceHttpUrl(brokerUrl.toString()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index d6b17f4faa4da..7c7dec864e2e8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1769,165 +1769,172 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { }); } - public CompletableFuture getManagedLedgerConfig(TopicName topicName) { + public CompletableFuture getManagedLedgerConfig(@Nonnull TopicName topicName) { + requireNonNull(topicName); NamespaceName namespace = topicName.getNamespaceObject(); ServiceConfiguration serviceConfig = pulsar.getConfiguration(); NamespaceResources nsr = pulsar.getPulsarResources().getNamespaceResources(); LocalPoliciesResources lpr = pulsar.getPulsarResources().getLocalPolicies(); - return nsr.getPoliciesAsync(namespace) - .thenCombine(lpr.getLocalPoliciesAsync(namespace), (policies, localPolicies) -> { - PersistencePolicies persistencePolicies = null; - RetentionPolicies retentionPolicies = null; - OffloadPoliciesImpl topicLevelOffloadPolicies = null; - - if (pulsar.getConfig().isTopicLevelPoliciesEnabled() - && !NamespaceService.isSystemServiceNamespace(namespace.toString())) { - final TopicPolicies topicPolicies = pulsar.getTopicPoliciesService() - .getTopicPoliciesIfExists(topicName); - if (topicPolicies != null) { - persistencePolicies = topicPolicies.getPersistence(); - retentionPolicies = topicPolicies.getRetentionPolicies(); - topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies(); - } - } - - if (persistencePolicies == null) { - persistencePolicies = policies.map(p -> p.persistence).orElseGet( - () -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), - serviceConfig.getManagedLedgerDefaultWriteQuorum(), - serviceConfig.getManagedLedgerDefaultAckQuorum(), - serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit())); - } + final CompletableFuture> topicPoliciesFuture; + if (pulsar.getConfig().isTopicLevelPoliciesEnabled() + && !NamespaceService.isSystemServiceNamespace(namespace.toString()) + && !SystemTopicNames.isTopicPoliciesSystemTopic(topicName.toString())) { + topicPoliciesFuture = pulsar.getTopicPoliciesService().getTopicPoliciesAsync(topicName); + } else { + topicPoliciesFuture = CompletableFuture.completedFuture(Optional.empty()); + } + return topicPoliciesFuture.thenCompose(topicPoliciesOptional -> { + final CompletableFuture> nsPolicies = nsr.getPoliciesAsync(namespace); + final CompletableFuture> lcPolicies = lpr.getLocalPoliciesAsync(namespace); + return nsPolicies.thenCombine(lcPolicies, (policies, localPolicies) -> { + PersistencePolicies persistencePolicies = null; + RetentionPolicies retentionPolicies = null; + OffloadPoliciesImpl topicLevelOffloadPolicies = null; + if (topicPoliciesOptional.isPresent()) { + final TopicPolicies topicPolicies = topicPoliciesOptional.get(); + persistencePolicies = topicPolicies.getPersistence(); + retentionPolicies = topicPolicies.getRetentionPolicies(); + topicLevelOffloadPolicies = topicPolicies.getOffloadPolicies(); + } - if (retentionPolicies == null) { - retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( - () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), - serviceConfig.getDefaultRetentionSizeInMB()) - ); - } + if (persistencePolicies == null) { + persistencePolicies = policies.map(p -> p.persistence).orElseGet( + () -> new PersistencePolicies(serviceConfig.getManagedLedgerDefaultEnsembleSize(), + serviceConfig.getManagedLedgerDefaultWriteQuorum(), + serviceConfig.getManagedLedgerDefaultAckQuorum(), + serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit())); + } - ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); - managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble()); - managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum()); - managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum()); + if (retentionPolicies == null) { + retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( + () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), + serviceConfig.getDefaultRetentionSizeInMB()) + ); + } - if (serviceConfig.isStrictBookieAffinityEnabled()) { + ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); + managedLedgerConfig.setEnsembleSize(persistencePolicies.getBookkeeperEnsemble()); + managedLedgerConfig.setWriteQuorumSize(persistencePolicies.getBookkeeperWriteQuorum()); + managedLedgerConfig.setAckQuorumSize(persistencePolicies.getBookkeeperAckQuorum()); + + if (serviceConfig.isStrictBookieAffinityEnabled()) { + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( + IsolatedBookieEnsemblePlacementPolicy.class); + if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } else if (isSystemTopic(topicName)) { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); + properties.put(IsolatedBookieEnsemblePlacementPolicy + .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } else { + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); + } + } else { + if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( IsolatedBookieEnsemblePlacementPolicy.class); - if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } else if (isSystemTopic(topicName)) { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, "*"); - properties.put(IsolatedBookieEnsemblePlacementPolicy - .SECONDARY_ISOLATION_BOOKIE_GROUPS, "*"); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } else { - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, ""); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, ""); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } - } else { - if (localPolicies.isPresent() && localPolicies.get().bookieAffinityGroup != null) { - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyClassName( - IsolatedBookieEnsemblePlacementPolicy.class); - Map properties = new HashMap<>(); - properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); - properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, - localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); - managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); - } + Map properties = new HashMap<>(); + properties.put(IsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupPrimary()); + properties.put(IsolatedBookieEnsemblePlacementPolicy.SECONDARY_ISOLATION_BOOKIE_GROUPS, + localPolicies.get().bookieAffinityGroup.getBookkeeperAffinityGroupSecondary()); + managedLedgerConfig.setBookKeeperEnsemblePlacementPolicyProperties(properties); } + } - managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate()); - managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType()); - managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword()); - - managedLedgerConfig - .setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist()); - managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( - serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); - managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( - serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); - managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); - managedLedgerConfig - .setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), - TimeUnit.MINUTES); - managedLedgerConfig - .setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), - TimeUnit.MINUTES); - managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes()); - - managedLedgerConfig.setMetadataOperationsTimeoutSeconds( - serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds()); - managedLedgerConfig - .setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds()); - managedLedgerConfig - .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); - managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); - managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( - serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); - managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); - managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); - managedLedgerConfig - .setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger()); - - managedLedgerConfig - .setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds()); - managedLedgerConfig - .setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); - managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); - managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData()); - managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery()); - managedLedgerConfig.setInactiveLedgerRollOverTime( - serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS); - managedLedgerConfig.setCacheEvictionByMarkDeletedPosition( - serviceConfig.isCacheEvictionByMarkDeletedPosition()); - managedLedgerConfig.setMinimumBacklogCursorsForCaching( - serviceConfig.getManagedLedgerMinimumBacklogCursorsForCaching()); - managedLedgerConfig.setMinimumBacklogEntriesForCaching( - serviceConfig.getManagedLedgerMinimumBacklogEntriesForCaching()); - managedLedgerConfig.setMaxBacklogBetweenCursorsForCaching( - serviceConfig.getManagedLedgerMaxBacklogBetweenCursorsForCaching()); - - OffloadPoliciesImpl nsLevelOffloadPolicies = - (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null); - OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration( - topicLevelOffloadPolicies, - OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), - getPulsar().getConfig().getProperties()); - if (NamespaceService.isSystemServiceNamespace(namespace.toString())) { - managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE); - } else { - if (topicLevelOffloadPolicies != null) { - try { - LedgerOffloader topicLevelLedgerOffLoader = - pulsar().createManagedLedgerOffloader(offloadPolicies); - managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); - } catch (PulsarServerException e) { - throw new RuntimeException(e); - } - } else { - //If the topic level policy is null, use the namespace level - managedLedgerConfig - .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); + managedLedgerConfig.setThrottleMarkDelete(persistencePolicies.getManagedLedgerMaxMarkDeleteRate()); + managedLedgerConfig.setDigestType(serviceConfig.getManagedLedgerDigestType()); + managedLedgerConfig.setPassword(serviceConfig.getManagedLedgerPassword()); + + managedLedgerConfig + .setMaxUnackedRangesToPersist(serviceConfig.getManagedLedgerMaxUnackedRangesToPersist()); + managedLedgerConfig.setPersistentUnackedRangesWithMultipleEntriesEnabled( + serviceConfig.isPersistentUnackedRangesWithMultipleEntriesEnabled()); + managedLedgerConfig.setMaxUnackedRangesToPersistInMetadataStore( + serviceConfig.getManagedLedgerMaxUnackedRangesToPersistInMetadataStore()); + managedLedgerConfig.setMaxEntriesPerLedger(serviceConfig.getManagedLedgerMaxEntriesPerLedger()); + managedLedgerConfig + .setMinimumRolloverTime(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), + TimeUnit.MINUTES); + managedLedgerConfig + .setMaximumRolloverTime(serviceConfig.getManagedLedgerMaxLedgerRolloverTimeMinutes(), + TimeUnit.MINUTES); + managedLedgerConfig.setMaxSizePerLedgerMb(serviceConfig.getManagedLedgerMaxSizePerLedgerMbytes()); + + managedLedgerConfig.setMetadataOperationsTimeoutSeconds( + serviceConfig.getManagedLedgerMetadataOperationsTimeoutSeconds()); + managedLedgerConfig + .setReadEntryTimeoutSeconds(serviceConfig.getManagedLedgerReadEntryTimeoutSeconds()); + managedLedgerConfig + .setAddEntryTimeoutSeconds(serviceConfig.getManagedLedgerAddEntryTimeoutSeconds()); + managedLedgerConfig.setMetadataEnsembleSize(serviceConfig.getManagedLedgerDefaultEnsembleSize()); + managedLedgerConfig.setUnackedRangesOpenCacheSetEnabled( + serviceConfig.isManagedLedgerUnackedRangesOpenCacheSetEnabled()); + managedLedgerConfig.setMetadataWriteQuorumSize(serviceConfig.getManagedLedgerDefaultWriteQuorum()); + managedLedgerConfig.setMetadataAckQuorumSize(serviceConfig.getManagedLedgerDefaultAckQuorum()); + managedLedgerConfig + .setMetadataMaxEntriesPerLedger(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger()); + + managedLedgerConfig + .setLedgerRolloverTimeout(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds()); + managedLedgerConfig + .setRetentionTime(retentionPolicies.getRetentionTimeInMinutes(), TimeUnit.MINUTES); + managedLedgerConfig.setRetentionSizeInMB(retentionPolicies.getRetentionSizeInMB()); + managedLedgerConfig.setAutoSkipNonRecoverableData(serviceConfig.isAutoSkipNonRecoverableData()); + managedLedgerConfig.setLazyCursorRecovery(serviceConfig.isLazyCursorRecovery()); + managedLedgerConfig.setInactiveLedgerRollOverTime( + serviceConfig.getManagedLedgerInactiveLedgerRolloverTimeSeconds(), TimeUnit.SECONDS); + managedLedgerConfig.setCacheEvictionByMarkDeletedPosition( + serviceConfig.isCacheEvictionByMarkDeletedPosition()); + managedLedgerConfig.setMinimumBacklogCursorsForCaching( + serviceConfig.getManagedLedgerMinimumBacklogCursorsForCaching()); + managedLedgerConfig.setMinimumBacklogEntriesForCaching( + serviceConfig.getManagedLedgerMinimumBacklogEntriesForCaching()); + managedLedgerConfig.setMaxBacklogBetweenCursorsForCaching( + serviceConfig.getManagedLedgerMaxBacklogBetweenCursorsForCaching()); + + OffloadPoliciesImpl nsLevelOffloadPolicies = + (OffloadPoliciesImpl) policies.map(p -> p.offload_policies).orElse(null); + OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.mergeConfiguration( + topicLevelOffloadPolicies, + OffloadPoliciesImpl.oldPoliciesCompatible(nsLevelOffloadPolicies, policies.orElse(null)), + getPulsar().getConfig().getProperties()); + if (NamespaceService.isSystemServiceNamespace(namespace.toString())) { + managedLedgerConfig.setLedgerOffloader(NullLedgerOffloader.INSTANCE); + } else { + if (topicLevelOffloadPolicies != null) { + try { + LedgerOffloader topicLevelLedgerOffLoader = + pulsar().createManagedLedgerOffloader(offloadPolicies); + managedLedgerConfig.setLedgerOffloader(topicLevelLedgerOffLoader); + } catch (PulsarServerException e) { + throw new RuntimeException(e); } + } else { + //If the topic level policy is null, use the namespace level + managedLedgerConfig + .setLedgerOffloader(pulsar.getManagedLedgerOffloader(namespace, offloadPolicies)); } + } - managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( - serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled()); - managedLedgerConfig.setNewEntriesCheckDelayInMillis( - serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis()); - return managedLedgerConfig; - }); + managedLedgerConfig.setDeletionAtBatchIndexLevelEnabled( + serviceConfig.isAcknowledgmentAtBatchIndexLevelEnabled()); + managedLedgerConfig.setNewEntriesCheckDelayInMillis( + serviceConfig.getManagedLedgerNewEntriesCheckDelayInMillis()); + return managedLedgerConfig; + }); + }); } private void addTopicToStatsMaps(TopicName topicName, Topic topic) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 9651e76a25e93..da31234095446 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -18,12 +18,14 @@ */ package org.apache.pulsar.broker.service; +import static java.util.Objects.requireNonNull; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -54,6 +56,7 @@ import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.FutureUtil; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,8 +81,8 @@ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesServic private final Map>> readerCaches = new ConcurrentHashMap<>(); - @VisibleForTesting - final Map policyCacheInitMap = new ConcurrentHashMap<>(); + + final Map> policyCacheInitMap = new ConcurrentHashMap<>(); @VisibleForTesting final Map>> listeners = new ConcurrentHashMap<>(); @@ -225,12 +228,12 @@ public TopicPolicies getTopicPolicies(TopicName topicName, } if (!policyCacheInitMap.containsKey(topicName.getNamespaceObject())) { NamespaceName namespace = topicName.getNamespaceObject(); - prepareInitPoliciesCache(namespace, new CompletableFuture<>()); + prepareInitPoliciesCacheAsync(namespace); } MutablePair result = new MutablePair<>(); policyCacheInitMap.compute(topicName.getNamespaceObject(), (k, initialized) -> { - if (initialized == null || !initialized) { + if (initialized == null || !initialized.isDone()) { result.setLeft(new TopicPoliciesCacheNotInitException()); } else { TopicPolicies topicPolicies = @@ -248,6 +251,34 @@ public TopicPolicies getTopicPolicies(TopicName topicName, } } + @NotNull + @Override + public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName, + boolean isGlobal) { + requireNonNull(topicName); + final CompletableFuture preparedFuture = prepareInitPoliciesCacheAsync(topicName.getNamespaceObject()); + return preparedFuture.thenApply(__ -> { + final TopicPolicies candidatePolicies = isGlobal + ? globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName())) + : policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); + return Optional.ofNullable(candidatePolicies); + }); + } + + @NotNull + @Override + public CompletableFuture> getTopicPoliciesAsync(@NotNull TopicName topicName) { + requireNonNull(topicName); + final CompletableFuture preparedFuture = prepareInitPoliciesCacheAsync(topicName.getNamespaceObject()); + return preparedFuture.thenApply(__ -> { + final TopicPolicies localPolicies = policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); + if (localPolicies != null) { + return Optional.of(localPolicies); + } + return Optional.ofNullable(globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName()))); + }); + } + @Override public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { return policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); @@ -271,39 +302,48 @@ public CompletableFuture getTopicPoliciesBypassCacheAsync(TopicNa @Override public CompletableFuture addOwnedNamespaceBundleAsync(NamespaceBundle namespaceBundle) { - CompletableFuture result = new CompletableFuture<>(); NamespaceName namespace = namespaceBundle.getNamespaceObject(); if (NamespaceService.isHeartbeatNamespace(namespace)) { - result.complete(null); - return result; + return CompletableFuture.completedFuture(null); } synchronized (this) { if (readerCaches.get(namespace) != null) { ownedBundlesCountPerNamespace.get(namespace).incrementAndGet(); - result.complete(null); + return CompletableFuture.completedFuture(null); } else { - prepareInitPoliciesCache(namespace, result); + return prepareInitPoliciesCacheAsync(namespace); } } - return result; } - private void prepareInitPoliciesCache(@Nonnull NamespaceName namespace, CompletableFuture result) { - if (policyCacheInitMap.putIfAbsent(namespace, false) == null) { - CompletableFuture> readerCompletableFuture = + private @Nonnull CompletableFuture prepareInitPoliciesCacheAsync(@Nonnull NamespaceName namespace) { + requireNonNull(namespace); + return policyCacheInitMap.computeIfAbsent(namespace, (k) -> { + final CompletableFuture> readerCompletableFuture = createSystemTopicClientWithRetry(namespace); readerCaches.put(namespace, readerCompletableFuture); ownedBundlesCountPerNamespace.putIfAbsent(namespace, new AtomicInteger(1)); - readerCompletableFuture.thenAccept(reader -> { - initPolicesCache(reader, result); - result.thenRun(() -> readMorePolicies(reader)); - }).exceptionally(ex -> { - log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); - cleanCacheAndCloseReader(namespace, false); - result.completeExceptionally(ex); + final CompletableFuture initFuture = readerCompletableFuture + .thenCompose(reader -> { + final CompletableFuture stageFuture = new CompletableFuture<>(); + initPolicesCache(reader, stageFuture); + return stageFuture + // Read policies in background + .thenAccept(__ -> readMorePoliciesAsync(reader)); + }); + initFuture.exceptionally(ex -> { + try { + log.error("[{}] Failed to create reader on __change_events topic", namespace, ex); + cleanCacheAndCloseReader(namespace, false); + } catch (Throwable cleanupEx) { + // Adding this catch to avoid break callback chain + log.error("[{}] Failed to cleanup reader on __change_events topic", namespace, cleanupEx); + } return null; }); - } + // let caller know we've got an exception. + return initFuture; + }); } protected CompletableFuture> createSystemTopicClientWithRetry( @@ -387,8 +427,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp if (log.isDebugEnabled()) { log.debug("[{}] Reach the end of the system topic.", reader.getSystemTopic().getTopicName()); } - policyCacheInitMap.computeIfPresent( - reader.getSystemTopic().getTopicName().getNamespaceObject(), (k, v) -> true); + // replay policy message policiesCache.forEach(((topicName, topicPolicies) -> { if (listeners.get(topicName) != null) { @@ -401,6 +440,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp } } })); + future.complete(null); } }); @@ -426,7 +466,13 @@ private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean }); } - private void readMorePolicies(SystemTopicClient.Reader reader) { + /** + * This is an async method for the background reader to continue syncing new messages. + * + * Note: You should not do any blocking call here. because it will affect + * #{@link SystemTopicBasedTopicPoliciesService#getTopicPoliciesAsync(TopicName)} method to block loading topic. + */ + private void readMorePoliciesAsync(SystemTopicClient.Reader reader) { reader.readNextAsync() .thenAccept(msg -> { refreshTopicPoliciesCache(msg); @@ -434,7 +480,7 @@ private void readMorePolicies(SystemTopicClient.Reader reader) { }) .whenComplete((__, ex) -> { if (ex == null) { - readMorePolicies(reader); + readMorePoliciesAsync(reader); } else { Throwable cause = FutureUtil.unwrapCompletionException(ex); if (cause instanceof PulsarClientException.AlreadyClosedException) { @@ -443,7 +489,7 @@ private void readMorePolicies(SystemTopicClient.Reader reader) { reader.getSystemTopic().getTopicName().getNamespaceObject(), false); } else { log.warn("Read more topic polices exception, read again.", ex); - readMorePolicies(reader); + readMorePoliciesAsync(reader); } } }); @@ -611,7 +657,7 @@ boolean checkReaderIsCached(NamespaceName namespaceName) { } @VisibleForTesting - public Boolean getPoliciesCacheInit(NamespaceName namespaceName) { + public CompletableFuture getPoliciesCacheInit(NamespaceName namespaceName) { return policyCacheInitMap.get(namespaceName); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java index c4bcc0c39353c..c09bab0a4b62c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicPoliciesService.java @@ -22,6 +22,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.BackoffBuilder; @@ -109,6 +110,32 @@ default CompletableFuture> getTopicPoliciesAsyncWithRetr return response; } + /** + * Asynchronously retrieves topic policies. + * This triggers the Pulsar broker's internal client to load policies from the + * system topic `persistent://tenant/namespace/__change_event`. + * + * @param topicName The name of the topic. + * @param isGlobal Indicates if the policies are global. + * @return A CompletableFuture containing an Optional of TopicPolicies. + * @throws NullPointerException If the topicName is null. + */ + @Nonnull + CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName, boolean isGlobal); + + /** + * Asynchronously retrieves topic policies. + * This triggers the Pulsar broker's internal client to load policies from the + * system topic `persistent://tenant/namespace/__change_event`. + * + * NOTE: If local policies are not available, it will fallback to using topic global policies. + * @param topicName The name of the topic. + * @return A CompletableFuture containing an Optional of TopicPolicies. + * @throws NullPointerException If the topicName is null. + */ + @Nonnull + CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName); + /** * Get policies for a topic without cache async. * @param topicName topic name @@ -162,6 +189,19 @@ public TopicPolicies getTopicPolicies(TopicName topicName, boolean isGlobal) return null; } + @Nonnull + @Override + public CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName, + boolean isGlobal) { + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Nonnull + @Override + public CompletableFuture> getTopicPoliciesAsync(@Nonnull TopicName topicName) { + return CompletableFuture.completedFuture(Optional.empty()); + } + @Override public TopicPolicies getTopicPoliciesIfExists(TopicName topicName) { return null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 284e50c830286..8dee90af4afb4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -126,6 +126,7 @@ public void initPersistentTopics() throws Exception { @Override @BeforeMethod protected void setup() throws Exception { + conf.setTopicLevelPoliciesEnabled(false); super.internalSetup(); persistentTopics = spy(PersistentTopics.class); persistentTopics.setServletContext(new MockServletContext()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 87471f4972f8d..faf141a5d1cf4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -180,7 +180,7 @@ public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Excep assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); //make sure namespace policy reader is fully started. Awaitility.await().untilAsserted(()-> { - assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject())); + assertTrue(policyService.getPoliciesCacheInit(topicName.getNamespaceObject()).isDone()); }); //load the topic. diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java new file mode 100644 index 0000000000000..672fc2c95f890 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesWithBrokerRestartTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Test(groups = "broker-admin") +public class TopicPoliciesWithBrokerRestartTest extends MockedPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true) + protected void setup() throws Exception { + super.internalSetup(); + setupDefaultTenantAndNamespace(); + } + + @Override + @AfterClass(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + + @Test + public void testRetentionWithBrokerRestart() throws Exception { + final int messages = 1_000; + final int topicNum = 500; + // (1) Init topic + admin.namespaces().createNamespace("public/retention"); + final String topicName = "persistent://public/retention/retention_with_broker_restart"; + admin.topics().createNonPartitionedTopic(topicName); + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.topics().createNonPartitionedTopic(shadowTopicNames); + } + // (2) Set retention + final RetentionPolicies retentionPolicies = new RetentionPolicies(20, 20); + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.topicPolicies().setRetention(shadowTopicNames, retentionPolicies); + } + admin.topicPolicies().setRetention(topicName, retentionPolicies); + // (3) Send messages + @Cleanup + final Producer publisher = pulsarClient.newProducer() + .topic(topicName) + .create(); + for (int i = 0; i < messages; i++) { + publisher.send((i + "").getBytes(StandardCharsets.UTF_8)); + } + // (4) Check configuration + Awaitility.await().untilAsserted(() -> { + final PersistentTopic persistentTopic1 = (PersistentTopic) + pulsar.getBrokerService().getTopic(topicName, true).join().get(); + final ManagedLedgerImpl managedLedger1 = (ManagedLedgerImpl) persistentTopic1.getManagedLedger(); + Assert.assertEquals(managedLedger1.getConfig().getRetentionSizeInMB(), 20); + Assert.assertEquals(managedLedger1.getConfig().getRetentionTimeMillis(), + TimeUnit.MINUTES.toMillis(20)); + }); + // (5) Restart broker + restartBroker(); + // (6) Check configuration again + for (int i = 0; i < topicNum; i++) { + final String shadowTopicNames = topicName + "_" + i; + admin.lookups().lookupTopic(shadowTopicNames); + final PersistentTopic persistentTopicTmp = (PersistentTopic) + pulsar.getBrokerService().getTopic(shadowTopicNames, true).join().get(); + final ManagedLedgerImpl managedLedgerTemp = (ManagedLedgerImpl) persistentTopicTmp.getManagedLedger(); + Assert.assertEquals(managedLedgerTemp.getConfig().getRetentionSizeInMB(), 20); + Assert.assertEquals(managedLedgerTemp.getConfig().getRetentionTimeMillis(), + TimeUnit.MINUTES.toMillis(20)); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java index efd8b66d754ac..234af7afa8d09 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsAuthTest.java @@ -84,6 +84,8 @@ protected void setup() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); super.internalSetup(); PulsarAdminBuilder pulsarAdminBuilder = PulsarAdmin.builder().serviceHttpUrl(brokerUrl != null ? brokerUrl.toString() : brokerUrlTls.toString()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java index 6ffcecbeb9f8b..942a42fa7aaa1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/AuthLogsTest.java @@ -60,6 +60,8 @@ public void setup() throws Exception { conf.setAuthorizationEnabled(true); conf.setAuthorizationAllowWildcardsMatching(true); conf.setSuperUserRoles(Sets.newHashSet("super")); + conf.setBrokerClientAuthenticationPlugin(MockAuthentication.class.getName()); + conf.setBrokerClientAuthenticationParameters("user:pass.pass"); internalSetup(); try (PulsarAdmin admin = PulsarAdmin.builder() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java index 0b1726617f71f..25ac59796b02c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockAuthentication.java @@ -29,7 +29,10 @@ public class MockAuthentication implements Authentication { private static final Logger log = LoggerFactory.getLogger(MockAuthentication.class); - private final String user; + private String user; + + public MockAuthentication() { + } public MockAuthentication(String user) { this.user = user; @@ -67,6 +70,7 @@ public String getCommandData() { @Override public void configure(Map authParams) { + this.user = authParams.get("user"); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java index 951892f4ebfbc..5252407892eea 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerBookieIsolationTest.java @@ -304,6 +304,7 @@ public void testSetRackInfoAndAffinityGroupDuringProduce() throws Exception { bookies[3].getBookieId()); ServiceConfiguration config = new ServiceConfiguration(); + config.setTopicLevelPoliciesEnabled(false); config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config.setClusterName(cluster); config.setWebServicePort(Optional.of(0)); @@ -612,9 +613,9 @@ public void testBookieIsolationWithSecondaryGroup() throws Exception { config.setBrokerShutdownTimeoutMs(0L); config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); + config.setTopicLevelPoliciesEnabled(false); config.setAdvertisedAddress("localhost"); config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups); - config.setManagedLedgerDefaultEnsembleSize(2); config.setManagedLedgerDefaultWriteQuorum(2); config.setManagedLedgerDefaultAckQuorum(2); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 31b5bcb23cd98..5b70ff996756e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -141,7 +141,7 @@ public void testGetPolicy() throws ExecutionException, InterruptedException, Top // Wait for all topic policies updated. Awaitility.await().untilAsserted(() -> Assert.assertTrue(systemTopicBasedTopicPoliciesService - .getPoliciesCacheInit(TOPIC1.getNamespaceObject()))); + .getPoliciesCacheInit(TOPIC1.getNamespaceObject()).isDone())); // Assert broker is cache all topic policies Awaitility.await().untilAsserted(() -> @@ -304,8 +304,8 @@ private void prepareData() throws PulsarAdminException { @Test public void testGetPolicyTimeout() throws Exception { SystemTopicBasedTopicPoliciesService service = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); - Awaitility.await().untilAsserted(() -> assertTrue(service.policyCacheInitMap.get(TOPIC1.getNamespaceObject()))); - service.policyCacheInitMap.put(TOPIC1.getNamespaceObject(), false); + Awaitility.await().untilAsserted(() -> assertTrue(service.policyCacheInitMap.get(TOPIC1.getNamespaceObject()).isDone())); + service.policyCacheInitMap.put(TOPIC1.getNamespaceObject(), new CompletableFuture<>()); long start = System.currentTimeMillis(); Backoff backoff = new BackoffBuilder() .setInitialTime(500, TimeUnit.MILLISECONDS) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 41704af0b8bb2..ac2727e33eb33 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -45,6 +45,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -620,7 +621,7 @@ public void testCheckPersistencePolicies() throws Exception { doReturn(policiesService).when(pulsar).getTopicPoliciesService(); TopicPolicies policies = new TopicPolicies(); policies.setRetentionPolicies(retentionPolicies); - doReturn(policies).when(policiesService).getTopicPoliciesIfExists(TopicName.get(topic)); + doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(policiesService).getTopicPoliciesAsync(TopicName.get(topic)); persistentTopic.onUpdate(policies); verify(persistentTopic, times(1)).checkPersistencePolicies(); Awaitility.await().untilAsserted(() -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java index f2631f591217b..2b6201fa56aba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthenticationTlsHostnameVerificationTest.java @@ -30,6 +30,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationTls; import org.apache.pulsar.common.tls.PublicSuffixMatcher; import org.apache.pulsar.common.tls.TlsHostnameVerifier; +import org.assertj.core.util.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -146,7 +147,10 @@ public void testTlsSyncProducerAndConsumerWithInvalidBrokerHost(boolean hostname clientTrustCertFilePath = TLS_MIM_TRUST_CERT_FILE_PATH; // setup broker cert which has CN = "pulsar" different than broker's hostname="localhost" conf.setBrokerServicePortTls(Optional.of(0)); + // disable topic level policy + conf.setTopicLevelPoliciesEnabled(false); conf.setWebServicePortTls(Optional.of(0)); + conf.setAuthenticationProviders(Sets.newTreeSet(AuthenticationProviderTls.class.getName())); conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_MIM_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_MIM_SERVER_KEY_FILE_PATH); @@ -188,9 +192,11 @@ public void testTlsSyncProducerAndConsumerCorrectBrokerHost() throws Exception { // setup broker cert which has CN = "localhost" conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); + conf.setAuthenticationProviders(Sets.newTreeSet(AuthenticationProviderTls.class.getName())); conf.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); + conf.setTopicLevelPoliciesEnabled(false); setup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java index 0ce3b7df07d1f..ba41848bf2c23 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/AuthorizationProducerConsumerTest.java @@ -119,6 +119,7 @@ protected void cleanup() throws Exception { public void testProducerAndConsumerAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(TestAuthorizationProvider.class.getName()); setup(); @@ -179,6 +180,7 @@ public void testProducerAndConsumerAuthorization() throws Exception { public void testSubscriberPermission() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setEnablePackagesManagement(true); conf.setPackagesManagementStorageProvider(MockedPackagesStorageProvider.class.getName()); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); @@ -369,6 +371,7 @@ public void testSubscriberPermission() throws Exception { public void testClearBacklogPermission() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); @@ -610,6 +613,7 @@ public void testUpdateTopicPropertiesAuthorization() throws Exception { public void testSubscriptionPrefixAuthorization() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(TestAuthorizationProviderWithSubscriptionPrefix.class.getName()); setup(); @@ -694,6 +698,7 @@ public void testAuthData() throws Exception { public void testPermissionForProducerCreateInitialSubscription() throws Exception { log.info("-- Starting {} test --", methodName); cleanup(); + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); setup(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java index 2fc8aebf64a4a..81d65b192049b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MutualAuthenticationTest.java @@ -195,7 +195,7 @@ protected void setup() throws Exception { Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); conf.setSuperUserRoles(superUserRoles); - + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthorizationEnabled(true); conf.setAuthenticationEnabled(true); Set providersClassNames = Sets.newHashSet(MutualAuthenticationProvider.class.getName()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java index 87f12e6acdcb2..4d5e7deaf7d99 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenAuthenticatedProducerConsumerTest.java @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.slf4j.Logger; @@ -92,6 +93,8 @@ protected void setup() throws Exception { Set providers = new HashSet<>(); providers.add(AuthenticationProviderToken.class.getName()); conf.setAuthenticationProviders(providers); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + ADMIN_TOKEN); conf.setClusterName("test"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java index fdf41c4a6ada1..ba43ee6d6a2dd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TokenOauth2AuthenticatedProducerConsumerTest.java @@ -27,7 +27,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -39,6 +41,7 @@ import org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,11 +90,12 @@ protected void setup() throws Exception { conf.setAuthenticationProviders(providers); conf.setBrokerClientAuthenticationPlugin(AuthenticationOAuth2.class.getName()); - conf.setBrokerClientAuthenticationParameters("{\n" - + " \"privateKey\": \"" + CREDENTIALS_FILE + "\",\n" - + " \"issuerUrl\": \"" + server.getIssuer() + "\",\n" - + " \"audience\": \"" + audience + "\",\n" - + "}\n"); + final Map oauth2Param = new HashMap<>(); + oauth2Param.put("privateKey", CREDENTIALS_FILE); + oauth2Param.put("issuerUrl", server.getIssuer()); + oauth2Param.put("audience", audience); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory + .getMapper().getObjectMapper().writeValueAsString(oauth2Param)); conf.setClusterName("test"); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java index 8e508b6cf2068..77405e142013a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/KeyStoreTlsProducerConsumerTestWithAuthTest.java @@ -32,6 +32,7 @@ import io.jsonwebtoken.SignatureAlgorithm; import lombok.Cleanup; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; @@ -49,6 +50,7 @@ import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.ObjectMapperFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -83,6 +85,7 @@ protected void cleanup() throws Exception { super.internalCleanup(); } + @SneakyThrows protected void internalSetUpForBroker() { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); @@ -114,6 +117,25 @@ protected void internalSetUpForBroker() { conf.setAuthenticationProviders(providers); conf.setNumExecutorThreadPoolSize(5); + Set tlsProtocols = Sets.newConcurrentHashSet(); + tlsProtocols.add("TLSv1.3"); + tlsProtocols.add("TLSv1.2"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationKeyStoreTls.class.getName()); + Map authParams = new HashMap<>(); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_TYPE, KEYSTORE_TYPE); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PATH, CLIENT_KEYSTORE_FILE_PATH); + authParams.put(AuthenticationKeyStoreTls.KEYSTORE_PW, CLIENT_KEYSTORE_PW); + conf.setBrokerClientAuthenticationParameters(ObjectMapperFactory.getMapper() + .getObjectMapper().writeValueAsString(authParams)); + conf.setBrokerClientTlsEnabled(true); + conf.setBrokerClientTlsEnabledWithKeyStore(true); + conf.setBrokerClientTlsTrustStore(BROKER_TRUSTSTORE_FILE_PATH); + conf.setBrokerClientTlsTrustStorePassword(BROKER_TRUSTSTORE_PW); + conf.setBrokerClientTlsKeyStore(CLIENT_KEYSTORE_FILE_PATH); + conf.setBrokerClientTlsKeyStoreType(KEYSTORE_TYPE); + conf.setBrokerClientTlsKeyStorePassword(CLIENT_KEYSTORE_PW); + conf.setBrokerClientTlsProtocols(tlsProtocols); + } protected void internalSetUpForClient(boolean addCertificates, String lookupUrl) throws Exception { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java index 76936334eb0ba..b9139dabdf021 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplAuthTest.java @@ -85,6 +85,7 @@ public void setup() throws Exception { // set isTcpLookup = true, to use BinaryProtoLookupService to get topics for a pattern. isTcpLookup = true; + conf.setTopicLevelPoliciesEnabled(false); conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); diff --git a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf index d9411e655ad5b..4e2fd40298354 100644 --- a/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf +++ b/pulsar-broker/src/test/resources/configurations/standalone_no_client_auth.conf @@ -29,4 +29,5 @@ authenticationEnabled=true authenticationProviders=org.apache.pulsar.MockTokenAuthenticationProvider brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= -loadBalancerOverrideBrokerNicSpeedGbps=2 \ No newline at end of file +loadBalancerOverrideBrokerNicSpeedGbps=2 +topicLevelPoliciesEnabled=false \ No newline at end of file diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java index 8229d929ee5e3..9c8e5197adf1a 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyAuthenticationTest.java @@ -168,7 +168,7 @@ protected void setup() throws Exception { conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); // Expires after an hour conf.setBrokerClientAuthenticationParameters( - "entityType:broker,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000)); + "entityType:admin,expiryTime:" + (System.currentTimeMillis() + 3600 * 1000)); Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java index 99af3b1cf6abe..b7cfb87474707 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyForwardAuthDataTest.java @@ -53,7 +53,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("authParam:broker"); + conf.setBrokerClientAuthenticationParameters("authParam:admin"); conf.setAuthenticateOriginalAuthData(true); Set superUserRoles = new HashSet(); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java index 2c8c382b6a5ef..3259cfd95c741 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyRolesEnforcementTest.java @@ -144,7 +144,7 @@ protected void setup() throws Exception { conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); conf.setBrokerClientAuthenticationPlugin(BasicAuthentication.class.getName()); - conf.setBrokerClientAuthenticationParameters("authParam:broker"); + conf.setBrokerClientAuthenticationParameters("authParam:admin"); Set superUserRoles = new HashSet<>(); superUserRoles.add("admin"); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java index e8bb128c8c190..2d97a4b06a856 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithAuthorizationNegTest.java @@ -78,6 +78,8 @@ public class ProxyWithAuthorizationNegTest extends ProducerConsumerBase { protected void setup() throws Exception { // enable tls and auth&auth at broker + conf.setTopicLevelPoliciesEnabled(false); + conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(true); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java index 6a61decfcc693..d47d09b8b85c0 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithoutServiceDiscoveryTest.java @@ -68,6 +68,9 @@ public class ProxyWithoutServiceDiscoveryTest extends ProducerConsumerBase { @Override protected void setup() throws Exception { + // Disable topic policy + conf.setTopicLevelPoliciesEnabled(false); + // enable tls and auth&auth at broker conf.setAuthenticationEnabled(true); conf.setAuthorizationEnabled(false); diff --git a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java index 9119ffed4e28f..7b550b7270f37 100644 --- a/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java +++ b/pulsar-sql/presto-pulsar/src/test/java/org/apache/pulsar/sql/presto/TestPulsarAuth.java @@ -38,6 +38,7 @@ import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.AuthenticationFactory; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; import org.apache.pulsar.common.policies.data.AuthAction; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; @@ -63,6 +64,9 @@ public void setup() throws Exception { conf.setProperties(properties); conf.setSuperUserRoles(Sets.newHashSet(SUPER_USER_ROLE)); conf.setClusterName("c1"); + conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + conf.setBrokerClientAuthenticationParameters("token:" + AuthTokenUtils + .createToken(secretKey, SUPER_USER_ROLE, Optional.empty())); internalSetup(); admin.clusters().createCluster("c1", ClusterData.builder().build()); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index 057039edc3be2..49e5ae378342d 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -89,6 +89,7 @@ public void setup() throws Exception { "org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder"); brokerEnvs.put("forceDeleteNamespaceAllowed", "true"); brokerEnvs.put("loadBalancerDebugModeEnabled", "true"); + brokerEnvs.put("topicLevelPoliciesEnabled", "false"); brokerEnvs.put("PULSAR_MEM", "-Xmx512M"); spec.brokerEnvs(brokerEnvs); pulsarCluster = PulsarCluster.forSpec(spec); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java index 0a9bb5e19592a..87db46f2bb625 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/presto/TestPulsarSQLAuth.java @@ -68,6 +68,7 @@ protected void beforeStartCluster() { envMap.put("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(secretKey)); envMap.put("superUserRoles", "admin"); envMap.put("brokerDeleteInactiveTopicsEnabled", "false"); + envMap.put("topicLevelPoliciesEnabled", "false"); for (BrokerContainer brokerContainer : pulsarCluster.getBrokers()) { brokerContainer.withEnv(envMap); From c6bffc2605d05c94eaca20d23502fb368926087f Mon Sep 17 00:00:00 2001 From: Kai Wang Date: Thu, 19 Oct 2023 07:08:49 -0500 Subject: [PATCH 333/494] [fix][broker] Fix unload operation stuck when use ExtensibleLoadManager (#21332) --- .../extensions/ExtensibleLoadManagerImpl.java | 2 +- .../channel/ServiceUnitStateChannelImpl.java | 17 +- .../extensions/manager/UnloadManager.java | 7 + .../broker/namespace/NamespaceService.java | 4 - .../pulsar/broker/service/BrokerService.java | 15 + .../ExtensibleLoadManagerImplTest.java | 311 ++++++++++-------- .../channel/ServiceUnitStateChannelTest.java | 12 +- 7 files changed, 219 insertions(+), 149 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 85baf9ec4fbdf..d3119365ddfea 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -304,7 +304,7 @@ public void start() throws PulsarServerException { } }); }); - this.serviceUnitStateChannel = new ServiceUnitStateChannelImpl(pulsar); + this.serviceUnitStateChannel = ServiceUnitStateChannelImpl.newInstance(pulsar); this.brokerRegistry.start(); this.splitManager = new SplitManager(splitCounter); this.unloadManager = new UnloadManager(unloadCounter); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 02acb923c2d94..68501d201f0d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -200,7 +200,18 @@ enum MetadataState { Unstable } + public static ServiceUnitStateChannelImpl newInstance(PulsarService pulsar) { + return new ServiceUnitStateChannelImpl(pulsar); + } + public ServiceUnitStateChannelImpl(PulsarService pulsar) { + this(pulsar, MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS, OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS); + } + + @VisibleForTesting + public ServiceUnitStateChannelImpl(PulsarService pulsar, + long inFlightStateWaitingTimeInMillis, + long ownershipMonitorDelayTimeInSecs) { this.pulsar = pulsar; this.config = pulsar.getConfig(); this.lookupServiceAddress = pulsar.getLookupServiceAddress(); @@ -210,8 +221,8 @@ public ServiceUnitStateChannelImpl(PulsarService pulsar) { this.stateChangeListeners = new StateChangeListeners(); this.semiTerminalStateWaitingTimeInMillis = config.getLoadBalancerServiceUnitStateTombstoneDelayTimeInSeconds() * 1000; - this.inFlightStateWaitingTimeInMillis = MAX_IN_FLIGHT_STATE_WAITING_TIME_IN_MILLIS; - this.ownershipMonitorDelayTimeInSecs = OWNERSHIP_MONITOR_DELAY_TIME_IN_SECS; + this.inFlightStateWaitingTimeInMillis = inFlightStateWaitingTimeInMillis; + this.ownershipMonitorDelayTimeInSecs = ownershipMonitorDelayTimeInSecs; if (semiTerminalStateWaitingTimeInMillis < inFlightStateWaitingTimeInMillis) { throw new IllegalArgumentException( "Invalid Config: loadBalancerServiceUnitStateCleanUpDelayTimeInSeconds < " @@ -837,7 +848,7 @@ private CompletableFuture deferGetOwnerRequest(String serviceUnit) { } finally { var future = requested.getValue(); if (future != null) { - future.orTimeout(inFlightStateWaitingTimeInMillis, TimeUnit.MILLISECONDS) + future.orTimeout(inFlightStateWaitingTimeInMillis + 5 * 1000, TimeUnit.MILLISECONDS) .whenComplete((v, e) -> { if (e != null) { getOwnerRequests.remove(serviceUnit, future); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java index 2dde0c4708e41..ffdbbc2af4219 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/manager/UnloadManager.java @@ -88,6 +88,13 @@ public CompletableFuture waitAsync(CompletableFuture eventPubFuture, @Override public void handleEvent(String serviceUnit, ServiceUnitStateData data, Throwable t) { + if (t != null && inFlightUnloadRequest.containsKey(serviceUnit)) { + if (log.isDebugEnabled()) { + log.debug("Handling {} for service unit {} with exception.", data, serviceUnit, t); + } + this.complete(serviceUnit, t); + return; + } ServiceUnitState state = ServiceUnitStateData.state(data); switch (state) { case Free, Owned -> this.complete(serviceUnit, t); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 0d69fd7ea7801..4c2b4211747c4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1544,10 +1544,6 @@ public boolean checkOwnershipPresent(NamespaceBundle bundle) throws Exception { public CompletableFuture checkOwnershipPresentAsync(NamespaceBundle bundle) { if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(config)) { - if (bundle.getNamespaceObject().equals(SYSTEM_NAMESPACE)) { - return FutureUtil.failedFuture(new UnsupportedOperationException( - "Ownership check for system namespace is not supported")); - } ExtensibleLoadManagerImpl extensibleLoadManager = ExtensibleLoadManagerImpl.get(loadManager.get()); return extensibleLoadManager.getOwnershipAsync(Optional.empty(), bundle) .thenApply(Optional::isPresent); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 7c7dec864e2e8..b5bd7bfaa3d51 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2202,6 +2202,21 @@ private CompletableFuture unloadServiceUnit(NamespaceBundle serviceUnit if (serviceUnit.includes(topicName)) { // Topic needs to be unloaded log.info("[{}] Unloading topic", topicName); + if (topicFuture.isCompletedExceptionally()) { + try { + topicFuture.get(); + } catch (InterruptedException | ExecutionException ex) { + if (ex.getCause() instanceof ServiceUnitNotReadyException) { + // Topic was already unloaded + if (log.isDebugEnabled()) { + log.debug("[{}] Topic was already unloaded", topicName); + } + return; + } else { + log.warn("[{}] Got exception when closing topic", topicName, ex); + } + } + } closeFutures.add(topicFuture .thenCompose(t -> t.isPresent() ? t.get().close(closeWithoutWaitingClientDisconnect) : CompletableFuture.completedFuture(null))); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 2d9de4b5e7f07..f499998fd3d6c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -38,9 +38,11 @@ import static org.apache.pulsar.broker.namespace.NamespaceService.getHeartbeatNamespace; import static org.apache.pulsar.broker.namespace.NamespaceService.getHeartbeatNamespaceV2; import static org.apache.pulsar.broker.namespace.NamespaceService.getSLAMonitorNamespace; +import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -111,7 +113,8 @@ import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; -import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.awaitility.Awaitility; +import org.mockito.MockedStatic; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -141,46 +144,56 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { @BeforeClass @Override public void setup() throws Exception { - conf.setForceDeleteNamespaceAllowed(true); - conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); - conf.setAllowAutoTopicCreation(true); - conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); - conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - conf.setLoadBalancerSheddingEnabled(false); - conf.setLoadBalancerDebugModeEnabled(true); - conf.setTopicLevelPoliciesEnabled(false); - super.internalSetup(conf); - pulsar1 = pulsar; - ServiceConfiguration defaultConf = getDefaultConf(); - defaultConf.setAllowAutoTopicCreation(true); - defaultConf.setForceDeleteNamespaceAllowed(true); - defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); - defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - defaultConf.setLoadBalancerSheddingEnabled(false); - defaultConf.setTopicLevelPoliciesEnabled(false); - additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); - pulsar2 = additionalPulsarTestContext.getPulsarService(); - - setPrimaryLoadManager(); - - setSecondaryLoadManager(); - - admin.clusters().createCluster(this.conf.getClusterName(), - ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); - admin.tenants().createTenant("public", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), - Sets.newHashSet(this.conf.getClusterName()))); - admin.namespaces().createNamespace("public/default"); - admin.namespaces().setNamespaceReplicationClusters("public/default", - Sets.newHashSet(this.conf.getClusterName())); - - admin.namespaces().createNamespace(defaultTestNamespace); - admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace, - Sets.newHashSet(this.conf.getClusterName())); + try (MockedStatic channelMockedStatic = + mockStatic(ServiceUnitStateChannelImpl.class)) { + channelMockedStatic.when(() -> ServiceUnitStateChannelImpl.newInstance(isA(PulsarService.class))) + .thenAnswer(invocation -> { + PulsarService pulsarService = invocation.getArgument(0); + // Set the inflight state waiting time and ownership monitor delay time to 5 seconds to avoid + // stuck when doing unload. + return new ServiceUnitStateChannelImpl(pulsarService, 5 * 1000, 1); + }); + conf.setForceDeleteNamespaceAllowed(true); + conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); + conf.setAllowAutoTopicCreation(true); + conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + conf.setLoadBalancerSheddingEnabled(false); + conf.setLoadBalancerDebugModeEnabled(true); + conf.setTopicLevelPoliciesEnabled(true); + super.internalSetup(conf); + pulsar1 = pulsar; + ServiceConfiguration defaultConf = getDefaultConf(); + defaultConf.setAllowAutoTopicCreation(true); + defaultConf.setForceDeleteNamespaceAllowed(true); + defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + defaultConf.setLoadBalancerSheddingEnabled(false); + defaultConf.setTopicLevelPoliciesEnabled(true); + additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); + pulsar2 = additionalPulsarTestContext.getPulsarService(); + + setPrimaryLoadManager(); + + setSecondaryLoadManager(); + + admin.clusters().createCluster(this.conf.getClusterName(), + ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + admin.tenants().createTenant("public", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), + Sets.newHashSet(this.conf.getClusterName()))); + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", + Sets.newHashSet(this.conf.getClusterName())); + + admin.namespaces().createNamespace(defaultTestNamespace); + admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace, + Sets.newHashSet(this.conf.getClusterName())); + } } @Override - @AfterClass + @AfterClass(alwaysRun = true) protected void cleanup() throws Exception { pulsar1 = null; pulsar2.close(); @@ -538,119 +551,134 @@ public CompletableFuture> filterAsync(Map webServiceUrl1 = - pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl1.isPresent()); - assertEquals(webServiceUrl1.get().toString(), pulsar3.getWebServiceAddress()); - - Optional webServiceUrl2 = - pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl2.isPresent()); - assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); - - Optional webServiceUrl3 = - pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl3.isPresent()); - assertEquals(webServiceUrl3.get().toString(), webServiceUrl1.get().toString()); - - List pulsarServices = List.of(pulsar1, pulsar2, pulsar3); - for (PulsarService pulsarService : pulsarServices) { - // Test lookup heartbeat namespace's topic - for (PulsarService pulsar : pulsarServices) { - assertLookupHeartbeatOwner(pulsarService, pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); - } - // Test lookup SLA namespace's topic - for (PulsarService pulsar : pulsarServices) { - assertLookupSLANamespaceOwner(pulsarService, pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); - } - } - - // Test deploy new broker with new load manager - ServiceConfiguration conf = getDefaultConf(); - conf.setAllowAutoTopicCreation(true); - conf.setForceDeleteNamespaceAllowed(true); - conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); - conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - try (var additionPulsarTestContext = createAdditionalPulsarTestContext(conf)) { - var pulsar4 = additionPulsarTestContext.getPulsarService(); - - Set availableCandidates = Sets.newHashSet(pulsar1.getBrokerServiceUrl(), - pulsar2.getBrokerServiceUrl(), - pulsar4.getBrokerServiceUrl()); - String lookupResult4 = pulsar4.getAdminClient().lookups().lookupTopic(topic); - assertTrue(availableCandidates.contains(lookupResult4)); - - String lookupResult5 = pulsar1.getAdminClient().lookups().lookupTopic(topic); - String lookupResult6 = pulsar2.getAdminClient().lookups().lookupTopic(topic); - String lookupResult7 = pulsar3.getAdminClient().lookups().lookupTopic(topic); - assertEquals(lookupResult4, lookupResult5); - assertEquals(lookupResult4, lookupResult6); - assertEquals(lookupResult4, lookupResult7); - - Set availableWebUrlCandidates = Sets.newHashSet(pulsar1.getWebServiceAddress(), - pulsar2.getWebServiceAddress(), - pulsar4.getWebServiceAddress()); - - webServiceUrl1 = + try (MockedStatic channelMockedStatic = + mockStatic(ServiceUnitStateChannelImpl.class)) { + channelMockedStatic.when(() -> ServiceUnitStateChannelImpl.newInstance(isA(PulsarService.class))) + .thenAnswer(invocation -> { + PulsarService pulsarService = invocation.getArgument(0); + // Set the inflight state waiting time and ownership monitor delay time to 5 seconds to avoid + // stuck when doing unload. + return new ServiceUnitStateChannelImpl(pulsarService, 5 * 1000, 1); + }); + // Test rollback to modular load manager. + ServiceConfiguration defaultConf = getDefaultConf(); + defaultConf.setAllowAutoTopicCreation(true); + defaultConf.setForceDeleteNamespaceAllowed(true); + defaultConf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); + defaultConf.setLoadBalancerSheddingEnabled(false); + try (var additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf)) { + // start pulsar3 with old load manager + var pulsar3 = additionalPulsarTestContext.getPulsarService(); + String topic = "persistent://" + defaultTestNamespace + "/test"; + + String lookupResult1 = pulsar3.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResult1, pulsar3.getBrokerServiceUrl()); + + String lookupResult2 = pulsar1.getAdminClient().lookups().lookupTopic(topic); + String lookupResult3 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResult1, lookupResult2); + assertEquals(lookupResult1, lookupResult3); + + NamespaceBundle bundle = getBundleAsync(pulsar1, TopicName.get(topic)).get(); + LookupOptions options = LookupOptions.builder() + .authoritative(false) + .requestHttps(false) + .readOnly(false) + .loadTopicsInBundle(false).build(); + Optional webServiceUrl1 = pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); assertTrue(webServiceUrl1.isPresent()); - assertTrue(availableWebUrlCandidates.contains(webServiceUrl1.get().toString())); + assertEquals(webServiceUrl1.get().toString(), pulsar3.getWebServiceAddress()); - webServiceUrl2 = + Optional webServiceUrl2 = pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); assertTrue(webServiceUrl2.isPresent()); assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); - // The pulsar3 will redirect to pulsar4 - webServiceUrl3 = + Optional webServiceUrl3 = pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); assertTrue(webServiceUrl3.isPresent()); - // It will redirect to pulsar4 - assertTrue(availableWebUrlCandidates.contains(webServiceUrl3.get().toString())); - - var webServiceUrl4 = - pulsar4.getNamespaceService().getWebServiceUrl(bundle, options); - assertTrue(webServiceUrl4.isPresent()); - assertEquals(webServiceUrl4.get().toString(), webServiceUrl1.get().toString()); + assertEquals(webServiceUrl3.get().toString(), webServiceUrl1.get().toString()); - pulsarServices = List.of(pulsar1, pulsar2, pulsar3, pulsar4); + List pulsarServices = List.of(pulsar1, pulsar2, pulsar3); for (PulsarService pulsarService : pulsarServices) { // Test lookup heartbeat namespace's topic for (PulsarService pulsar : pulsarServices) { - assertLookupHeartbeatOwner(pulsarService, pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + assertLookupHeartbeatOwner(pulsarService, + pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); } // Test lookup SLA namespace's topic for (PulsarService pulsar : pulsarServices) { - assertLookupSLANamespaceOwner(pulsarService, pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + assertLookupSLANamespaceOwner(pulsarService, + pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + } + } + + // Test deploy new broker with new load manager + ServiceConfiguration conf = getDefaultConf(); + conf.setAllowAutoTopicCreation(true); + conf.setForceDeleteNamespaceAllowed(true); + conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + try (var additionPulsarTestContext = createAdditionalPulsarTestContext(conf)) { + var pulsar4 = additionPulsarTestContext.getPulsarService(); + + Set availableCandidates = Sets.newHashSet(pulsar1.getBrokerServiceUrl(), + pulsar2.getBrokerServiceUrl(), + pulsar4.getBrokerServiceUrl()); + String lookupResult4 = pulsar4.getAdminClient().lookups().lookupTopic(topic); + assertTrue(availableCandidates.contains(lookupResult4)); + + String lookupResult5 = pulsar1.getAdminClient().lookups().lookupTopic(topic); + String lookupResult6 = pulsar2.getAdminClient().lookups().lookupTopic(topic); + String lookupResult7 = pulsar3.getAdminClient().lookups().lookupTopic(topic); + assertEquals(lookupResult4, lookupResult5); + assertEquals(lookupResult4, lookupResult6); + assertEquals(lookupResult4, lookupResult7); + + Set availableWebUrlCandidates = Sets.newHashSet(pulsar1.getWebServiceAddress(), + pulsar2.getWebServiceAddress(), + pulsar4.getWebServiceAddress()); + + webServiceUrl1 = + pulsar1.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl1.isPresent()); + assertTrue(availableWebUrlCandidates.contains(webServiceUrl1.get().toString())); + + webServiceUrl2 = + pulsar2.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl2.isPresent()); + assertEquals(webServiceUrl2.get().toString(), webServiceUrl1.get().toString()); + + // The pulsar3 will redirect to pulsar4 + webServiceUrl3 = + pulsar3.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl3.isPresent()); + // It will redirect to pulsar4 + assertTrue(availableWebUrlCandidates.contains(webServiceUrl3.get().toString())); + + var webServiceUrl4 = + pulsar4.getNamespaceService().getWebServiceUrl(bundle, options); + assertTrue(webServiceUrl4.isPresent()); + assertEquals(webServiceUrl4.get().toString(), webServiceUrl1.get().toString()); + + pulsarServices = List.of(pulsar1, pulsar2, pulsar3, pulsar4); + for (PulsarService pulsarService : pulsarServices) { + // Test lookup heartbeat namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupHeartbeatOwner(pulsarService, + pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + } + // Test lookup SLA namespace's topic + for (PulsarService pulsar : pulsarServices) { + assertLookupSLANamespaceOwner(pulsarService, + pulsar.getLookupServiceAddress(), pulsar.getBrokerServiceUrl()); + } } } } } + } private void assertLookupHeartbeatOwner(PulsarService pulsar, @@ -1089,6 +1117,12 @@ public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws Exceptio NamespaceName heartbeatNamespacePulsar2V2 = NamespaceService.getHeartbeatNamespaceV2(pulsar2.getLookupServiceAddress(), pulsar2.getConfiguration()); + NamespaceName slaMonitorNamespacePulsar1 = + getSLAMonitorNamespace(pulsar1.getLookupServiceAddress(), pulsar1.getConfiguration()); + + NamespaceName slaMonitorNamespacePulsar2 = + getSLAMonitorNamespace(pulsar2.getLookupServiceAddress(), pulsar2.getConfiguration()); + NamespaceBundle bundle1 = pulsar1.getNamespaceService().getNamespaceBundleFactory() .getFullBundle(heartbeatNamespacePulsar1V1); NamespaceBundle bundle2 = pulsar1.getNamespaceService().getNamespaceBundleFactory() @@ -1099,27 +1133,34 @@ public void testGetOwnedServiceUnitsAndGetOwnedNamespaceStatus() throws Exceptio NamespaceBundle bundle4 = pulsar2.getNamespaceService().getNamespaceBundleFactory() .getFullBundle(heartbeatNamespacePulsar2V2); + NamespaceBundle slaBundle1 = pulsar1.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(slaMonitorNamespacePulsar1); + NamespaceBundle slaBundle2 = pulsar2.getNamespaceService().getNamespaceBundleFactory() + .getFullBundle(slaMonitorNamespacePulsar2); + + Set ownedServiceUnitsByPulsar1 = primaryLoadManager.getOwnedServiceUnits(); log.info("Owned service units: {}", ownedServiceUnitsByPulsar1); // heartbeat namespace bundle will own by pulsar1 - assertEquals(ownedServiceUnitsByPulsar1.size(), 3); assertTrue(ownedServiceUnitsByPulsar1.contains(bundle1)); assertTrue(ownedServiceUnitsByPulsar1.contains(bundle2)); + assertTrue(ownedServiceUnitsByPulsar1.contains(slaBundle1)); Set ownedServiceUnitsByPulsar2 = secondaryLoadManager.getOwnedServiceUnits(); log.info("Owned service units: {}", ownedServiceUnitsByPulsar2); - assertEquals(ownedServiceUnitsByPulsar2.size(), 3); assertTrue(ownedServiceUnitsByPulsar2.contains(bundle3)); assertTrue(ownedServiceUnitsByPulsar2.contains(bundle4)); + assertTrue(ownedServiceUnitsByPulsar2.contains(slaBundle2)); Map ownedNamespacesByPulsar1 = admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar1.getLookupServiceAddress()); Map ownedNamespacesByPulsar2 = admin.brokers().getOwnedNamespaces(conf.getClusterName(), pulsar2.getLookupServiceAddress()); - assertEquals(ownedNamespacesByPulsar1.size(), 3); assertTrue(ownedNamespacesByPulsar1.containsKey(bundle1.toString())); assertTrue(ownedNamespacesByPulsar1.containsKey(bundle2.toString())); - assertEquals(ownedNamespacesByPulsar2.size(), 3); + assertTrue(ownedNamespacesByPulsar1.containsKey(slaBundle1.toString())); + assertTrue(ownedNamespacesByPulsar2.containsKey(bundle3.toString())); assertTrue(ownedNamespacesByPulsar2.containsKey(bundle4.toString())); + assertTrue(ownedNamespacesByPulsar2.containsKey(slaBundle2.toString())); String topic = "persistent://" + defaultTestNamespace + "/test-get-owned-service-units"; admin.topics().createPartitionedTopic(topic, 1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 990408d214b7f..e8ccd7b01ca55 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -507,10 +507,10 @@ public void transferTestWhenDestBrokerFails() assertEquals(1, getOwnerRequests1.size()); assertEquals(1, getOwnerRequests2.size()); - // In 5 secs, the getOwnerAsync requests(lookup requests) should time out. - Awaitility.await().atMost(5, TimeUnit.SECONDS) + // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. + Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); - Awaitility.await().atMost(5, TimeUnit.SECONDS) + Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); assertEquals(0, getOwnerRequests1.size()); @@ -1139,10 +1139,10 @@ public void assignTestWhenDestBrokerProducerFails() assertFalse(owner1.isDone()); assertFalse(owner2.isDone()); - // In 5 secs, the getOwnerAsync requests(lookup requests) should time out. - Awaitility.await().atMost(5, TimeUnit.SECONDS) + // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. + Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); - Awaitility.await().atMost(5, TimeUnit.SECONDS) + Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); // recovered, check the monitor update state : Assigned -> Owned From beed0a34488d80657e52694ce3b980fe40fe3ffc Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 17 Oct 2023 09:38:02 +0800 Subject: [PATCH 334/494] [fix][sec] Bump avro version to 1.11.3 for CVE-2023-39410 (#21341) Signed-off-by: tison (cherry picked from commit f5222d6b1f64d14029f8fef3e0108cf74ffefded) --- distribution/server/src/assemble/LICENSE.bin.txt | 4 ++-- distribution/shell/src/assemble/LICENSE.bin.txt | 4 ++-- pom.xml | 2 +- .../schema/compatibility/SchemaCompatibilityCheckTest.java | 2 +- .../pulsar/client/impl/schema/ProtobufSchemaTest.java | 6 +++--- pulsar-io/kafka-connect-adaptor/pom.xml | 6 ++++++ pulsar-sql/presto-distribution/LICENSE | 4 ++-- 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index cdea68ac69049..de724732e4a1e 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -448,8 +448,8 @@ The Apache Software License, Version 2.0 - net.jodah-typetools-0.5.0.jar - net.jodah-failsafe-2.4.4.jar * Apache Avro - - org.apache.avro-avro-1.10.2.jar - - org.apache.avro-avro-protobuf-1.10.2.jar + - org.apache.avro-avro-1.11.3.jar + - org.apache.avro-avro-protobuf-1.11.3.jar * Apache Curator - org.apache.curator-curator-client-5.1.0.jar - org.apache.curator-curator-framework-5.1.0.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 6fc5442cf7a18..e69b74e7a9ebf 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -411,8 +411,8 @@ The Apache Software License, Version 2.0 * Google Error Prone Annotations - error_prone_annotations-2.5.1.jar * Javassist -- javassist-3.25.0-GA.jar * Apache Avro - - avro-1.10.2.jar - - avro-protobuf-1.10.2.jar + - avro-1.11.3.jar + - avro-protobuf-1.11.3.jar BSD 3-clause "New" or "Revised" License * JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt diff --git a/pom.xml b/pom.xml index efb0e8570ee54..386be8cb6eae6 100644 --- a/pom.xml +++ b/pom.xml @@ -177,7 +177,7 @@ flexible messaging model and an intuitive client API. 3.4.0 5.5.3 1.12.262 - 1.10.2 + 1.11.3 2.10.10 2.5.0 5.1.0 diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java index 140dea9e7ebc7..49517a424b936 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/compatibility/SchemaCompatibilityCheckTest.java @@ -407,7 +407,7 @@ public void testSchemaComparison() throws Exception { assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()), SchemaCompatibilityStrategy.UNDEFINED); byte[] changeSchemaBytes = (new String(Schema.AVRO(Schemas.PersonOne.class) - .getSchemaInfo().getSchema(), UTF_8) + "/n /n /n").getBytes(); + .getSchemaInfo().getSchema(), UTF_8) + "\n \n \n").getBytes(); SchemaInfo schemaInfo = SchemaInfo.builder().type(SchemaType.AVRO).schema(changeSchemaBytes).build(); admin.schemas().createSchema(fqtn, schemaInfo); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java index 3fcd6f12b982d..85012276d5af1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java @@ -41,20 +41,20 @@ public class ProtobufSchemaTest { "\"namespace\":\"org.apache.pulsar.client.schema.proto.Test\"," + "\"fields\":[{\"name\":\"stringField\",\"type\":{\"type\":\"string\"," + "\"avro.java.string\":\"String\"},\"default\":\"\"},{\"name\":\"doubleField\"," + - "\"type\":\"double\",\"default\":0},{\"name\":\"intField\",\"type\":\"int\"," + + "\"type\":\"double\",\"default\":0.0},{\"name\":\"intField\",\"type\":\"int\"," + "\"default\":0},{\"name\":\"testEnum\",\"type\":{\"type\":\"enum\"," + "\"name\":\"TestEnum\",\"symbols\":[\"SHARED\",\"FAILOVER\"]}," + "\"default\":\"SHARED\"},{\"name\":\"nestedField\"," + "\"type\":[\"null\",{\"type\":\"record\",\"name\":\"SubMessage\"," + "\"fields\":[{\"name\":\"foo\",\"type\":{\"type\":\"string\"," + "\"avro.java.string\":\"String\"},\"default\":\"\"}" + - ",{\"name\":\"bar\",\"type\":\"double\",\"default\":0}]}]" + + ",{\"name\":\"bar\",\"type\":\"double\",\"default\":0.0}]}]" + ",\"default\":null},{\"name\":\"repeatedField\",\"type\":{\"type\":\"array\"" + ",\"items\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},\"default\":[]}" + ",{\"name\":\"externalMessage\",\"type\":[\"null\",{\"type\":\"record\"" + ",\"name\":\"ExternalMessage\",\"namespace\":\"org.apache.pulsar.client.schema.proto.ExternalTest\"" + ",\"fields\":[{\"name\":\"stringField\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}," + - "\"default\":\"\"},{\"name\":\"doubleField\",\"type\":\"double\",\"default\":0}]}],\"default\":null}]}"; + "\"default\":\"\"},{\"name\":\"doubleField\",\"type\":\"double\",\"default\":0.0}]}],\"default\":null}]}"; private static final String EXPECTED_PARSING_INFO = "{\"__alwaysAllowNull\":\"true\",\"__jsr310ConversionEnabled\":\"false\"," + "\"__PARSING_INFO__\":\"[{\\\"number\\\":1,\\\"name\\\":\\\"stringField\\\",\\\"type\\\":\\\"STRING\\\"," + diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index a2159478eddd1..3d10a31a6cd2e 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -116,6 +116,12 @@ io.confluent kafka-connect-avro-converter ${confluent.version} + + + org.apache.avro + avro + + diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index ca117edc5341c..e8e9aa0e708b1 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -368,8 +368,8 @@ The Apache Software License, Version 2.0 * OpenCSV - opencsv-2.3.jar * Avro - - avro-1.10.2.jar - - avro-protobuf-1.10.2.jar + - avro-1.11.3.jar + - avro-protobuf-1.11.3.jar * Caffeine - caffeine-2.9.1.jar * Javax From 77d20857911fcfc5f8414845b994716d0859b1a7 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Wed, 17 May 2023 04:14:44 -0500 Subject: [PATCH 335/494] [feat][meta] Upgrade to jetcd to 0.7.5 (#20339) (cherry picked from commit c7a4060763b905925ffe8e9690ea983d863aa52f) --- .../server/src/assemble/LICENSE.bin.txt | 7 +++++-- pom.xml | 2 +- .../pulsar/metadata/BaseMetadataStoreTest.java | 7 ++++--- .../impl/LeaderElectionImplTest.java | 2 +- .../metadata/impl/EtcdMetadataStoreTest.java | 17 ++++++++++------- pulsar-sql/presto-distribution/LICENSE | 12 ++++++++---- 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index de724732e4a1e..7a993e00615e0 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -479,6 +479,7 @@ The Apache Software License, Version 2.0 - io.vertx-vertx-core-4.3.8.jar - io.vertx-vertx-web-4.3.8.jar - io.vertx-vertx-web-common-4.3.8.jar + - io.vertx-vertx-grpc-4.3.8.jar * Apache ZooKeeper - org.apache.zookeeper-zookeeper-3.8.3.jar - org.apache.zookeeper-zookeeper-jute-3.8.3.jar @@ -491,8 +492,10 @@ The Apache Software License, Version 2.0 - com.google.auto.value-auto-value-annotations-1.9.jar - com.google.re2j-re2j-1.6.jar * Jetcd - - io.etcd-jetcd-common-0.5.11.jar - - io.etcd-jetcd-core-0.5.11.jar + - io.etcd-jetcd-api-0.7.5.jar + - io.etcd-jetcd-common-0.7.5.jar + - io.etcd-jetcd-core-0.7.5.jar + - io.etcd-jetcd-grpc-0.7.5.jar * IPAddress - com.github.seancfoley-ipaddress-5.3.3.jar * RxJava diff --git a/pom.xml b/pom.xml index 386be8cb6eae6..f25f1428c1ce8 100644 --- a/pom.xml +++ b/pom.xml @@ -242,7 +242,7 @@ flexible messaging model and an intuitive client API. 5.3.27 4.5.13 4.4.15 - 0.5.11 + 0.7.5 2.0 1.10.12 5.3.3 diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java index 57a4e388572fa..ec6e6e03eae71 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/BaseMetadataStoreTest.java @@ -21,7 +21,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import io.etcd.jetcd.launcher.EtcdCluster; -import io.etcd.jetcd.launcher.EtcdClusterFactory; +import io.etcd.jetcd.test.EtcdClusterExtension; import java.io.File; import java.net.URI; import java.util.UUID; @@ -84,10 +84,11 @@ public Object[][] implementations() { private synchronized String getEtcdClusterConnectString() { if (etcdCluster == null) { - etcdCluster = EtcdClusterFactory.buildCluster("test", 1, false); + etcdCluster = EtcdClusterExtension.builder().withClusterName("test").withNodes(1).withSsl(false).build() + .cluster(); etcdCluster.start(); } - return etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + return etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); } public static Supplier stringSupplier(Supplier supplier) { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java index 027521d2ffc17..09c9d71c41a30 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/coordination/impl/LeaderElectionImplTest.java @@ -29,7 +29,7 @@ public class LeaderElectionImplTest extends BaseMetadataStoreTest { - @Test(dataProvider = "impl", timeOut = 10000) + @Test(dataProvider = "impl", timeOut = 20000) public void validateDeadLock(String provider, Supplier urlSupplier) throws Exception { if (provider.equals("Memory") || provider.equals("RocksDB")) { diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java index f1a1b5626acce..bdcd0614d0375 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/EtcdMetadataStoreTest.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.io.Resources; import io.etcd.jetcd.launcher.EtcdCluster; -import io.etcd.jetcd.launcher.EtcdClusterFactory; +import io.etcd.jetcd.test.EtcdClusterExtension; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -44,7 +44,8 @@ public class EtcdMetadataStoreTest { @Test public void testCluster() throws Exception { @Cleanup - EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-cluster", 3, false); + EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-cluster").withNodes(3) + .withSsl(false).build().cluster(); etcdCluster.start(); EtcdConfig etcdConfig = EtcdConfig.builder().useTls(false) @@ -56,7 +57,7 @@ public void testCluster() throws Exception { new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig); String metadataURL = - "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); @Cleanup MetadataStore store = MetadataStoreFactory.create(metadataURL, @@ -71,7 +72,8 @@ public void testCluster() throws Exception { @Test public void testClusterWithTls() throws Exception { @Cleanup - EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-cluster", 3, true); + EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-cluster").withNodes(3) + .withSsl(true).build().cluster(); etcdCluster.start(); EtcdConfig etcdConfig = EtcdConfig.builder().useTls(true) @@ -86,7 +88,7 @@ public void testClusterWithTls() throws Exception { new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig); String metadataURL = - "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); @Cleanup MetadataStore store = MetadataStoreFactory.create(metadataURL, @@ -101,7 +103,8 @@ public void testClusterWithTls() throws Exception { @Test public void testTlsInstance() throws Exception { @Cleanup - EtcdCluster etcdCluster = EtcdClusterFactory.buildCluster("test-tls", 1, true); + EtcdCluster etcdCluster = EtcdClusterExtension.builder().withClusterName("test-tls").withNodes(1) + .withSsl(true).build().cluster(); etcdCluster.start(); EtcdConfig etcdConfig = EtcdConfig.builder().useTls(true) @@ -115,7 +118,7 @@ public void testTlsInstance() throws Exception { new ObjectMapper(new YAMLFactory()).writeValue(etcdConfigPath.toFile(), etcdConfig); String metadataURL = - "etcd:" + etcdCluster.getClientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); + "etcd:" + etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")); @Cleanup MetadataStore store = MetadataStoreFactory.create(metadataURL, diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index e8e9aa0e708b1..11ba45b2bc740 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -272,9 +272,13 @@ The Apache Software License, Version 2.0 - grpc-protobuf-lite-1.55.3.jar - grpc-stub-1.55.3.jar * JEtcd - - jetcd-common-0.5.11.jar - - jetcd-core-0.5.11.jar - + - jetcd-api-0.7.5.jar + - jetcd-common-0.7.5.jar + - jetcd-core-0.7.5.jar + - jetcd-grpc-0.7.5.jar + * Vertx + - vertx-core-4.3.8.jar + - vertx-grpc-4.3.5.jar * Joda Time - joda-time-2.10.10.jar - failsafe-2.4.4.jar @@ -521,7 +525,7 @@ MIT License * Checker Qual - checker-qual-3.33.0.jar * Annotations - - animal-sniffer-annotations-1.19.jar + - animal-sniffer-annotations-1.21.jar - annotations-4.1.1.4.jar * ScribeJava - scribejava-apis-6.9.0.jar From 0923080070e234260f8a015bbbaac1d4021d4d72 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 3 Oct 2023 14:06:05 +0300 Subject: [PATCH 336/494] [fix][sec] Upgrade snappy-java to 1.1.10.5 (#21280) (cherry picked from commit 643428bb295ee4781e86aa2c72e5ad5d61b98870) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 7a993e00615e0..fe5a3989b16f5 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -485,7 +485,7 @@ The Apache Software License, Version 2.0 - org.apache.zookeeper-zookeeper-jute-3.8.3.jar - org.apache.zookeeper-zookeeper-prometheus-metrics-3.8.3.jar * Snappy Java - - org.xerial.snappy-snappy-java-1.1.10.1.jar + - org.xerial.snappy-snappy-java-1.1.10.5.jar * Google HTTP Client - com.google.http-client-google-http-client-gson-1.41.0.jar - com.google.http-client-google-http-client-1.41.0.jar diff --git a/pom.xml b/pom.xml index f25f1428c1ce8..33176ee70635d 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ flexible messaging model and an intuitive client API. 3.8.3 1.5.0 1.10.0 - 1.1.10.1 + 1.1.10.5 4.1.12.1 5.1.0 4.1.100.Final diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 11ba45b2bc740..d26cd5f80f987 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -458,7 +458,7 @@ The Apache Software License, Version 2.0 * JSON Simple - json-simple-1.1.1.jar * Snappy - - snappy-java-1.1.10.1.jar + - snappy-java-1.1.10.5.jar * Jackson - jackson-module-parameter-names-2.14.2.jar * Java Assist From df4b0d8ff168979e4a64d09245c18cf74b9d46f5 Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Tue, 24 Oct 2023 19:08:21 +0800 Subject: [PATCH 337/494] [fix][proxy] Move status endpoint out of auth coverage (#21428) (cherry picked from commit fe2d61d5a44344042ec1994d0943cfc7977fbdcd) --- .../proxy/server/ProxyServiceStarter.java | 6 +++-- .../apache/pulsar/proxy/server/WebServer.java | 27 ++++++++++++++++++- .../server/ProxyWithJwtAuthorizationTest.java | 24 +++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index d0774cee88301..84f83a901a3e4 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -255,9 +255,11 @@ public static void addWebServerHandlers(WebServer server, ProxyConfiguration config, ProxyService service, BrokerDiscoveryProvider discoveryProvider) throws Exception { + // We can make 'status.html' publicly accessible without authentication since + // it does not contain any sensitive data. + server.addRestResource("/", VipStatus.ATTRIBUTE_STATUS_FILE_PATH, config.getStatusFilePath(), + VipStatus.class, false); if (config.isEnableProxyStatsEndpoints()) { - server.addRestResource("/", VipStatus.ATTRIBUTE_STATUS_FILE_PATH, config.getStatusFilePath(), - VipStatus.class); server.addRestResource("/proxy-stats", ProxyStats.ATTRIBUTE_PULSAR_PROXY_NAME, service, ProxyStats.class); if (service != null) { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java index edbcfe0847c4e..b95bbcab08b11 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java @@ -239,7 +239,31 @@ private static void popularServletParams(ServletHolder servletHolder, ProxyConfi } } + /** + * Add a REST resource to the servlet context with authentication coverage. + * + * @see WebServer#addRestResource(String, String, Object, Class, boolean) + * + * @param basePath The base path for the resource. + * @param attribute An attribute associated with the resource. + * @param attributeValue The value of the attribute. + * @param resourceClass The class representing the resource. + */ public void addRestResource(String basePath, String attribute, Object attributeValue, Class resourceClass) { + addRestResource(basePath, attribute, attributeValue, resourceClass, true); + } + + /** + * Add a REST resource to the servlet context. + * + * @param basePath The base path for the resource. + * @param attribute An attribute associated with the resource. + * @param attributeValue The value of the attribute. + * @param resourceClass The class representing the resource. + * @param requireAuthentication A boolean indicating whether authentication is required for this resource. + */ + public void addRestResource(String basePath, String attribute, Object attributeValue, + Class resourceClass, boolean requireAuthentication) { ResourceConfig config = new ResourceConfig(); config.register(resourceClass); config.register(JsonMapperProvider.class); @@ -247,7 +271,8 @@ public void addRestResource(String basePath, String attribute, Object attributeV servletHolder.setAsyncSupported(true); // This method has not historically checked for existing paths, so we don't check here either. The // method call is added to reduce code duplication. - addServlet(basePath, servletHolder, Collections.singletonList(Pair.of(attribute, attributeValue)), true, false); + addServlet(basePath, servletHolder, Collections.singletonList(Pair.of(attribute, attributeValue)), + requireAuthentication, false); } public int getExternalServicePort() { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java index e912006faa022..88ecfe8a3187b 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyWithJwtAuthorizationTest.java @@ -116,6 +116,7 @@ protected void setup() throws Exception { proxyConfig.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); proxyConfig.setBrokerClientAuthenticationParameters(PROXY_TOKEN); proxyConfig.setAuthenticationProviders(providers); + proxyConfig.setStatusFilePath("./src/test/resources/vip_status.html"); AuthenticationService authService = new AuthenticationService(PulsarConfigurationLoader.convertFrom(proxyConfig)); @@ -405,6 +406,29 @@ public void testProxyAuthorizationWithPrefixSubscriptionAuthMode() throws Except log.info("-- Exiting {} test --", methodName); } + @Test + void testGetStatus() throws Exception { + log.info("-- Starting {} test --", methodName); + final PulsarResources resource = new PulsarResources(new ZKMetadataStore(mockZooKeeper), + new ZKMetadataStore(mockZooKeeperGlobal)); + final AuthenticationService authService = new AuthenticationService( + PulsarConfigurationLoader.convertFrom(proxyConfig)); + final WebServer webServer = new WebServer(proxyConfig, authService); + ProxyServiceStarter.addWebServerHandlers(webServer, proxyConfig, proxyService, + new BrokerDiscoveryProvider(proxyConfig, resource)); + webServer.start(); + @Cleanup + final Client client = javax.ws.rs.client.ClientBuilder + .newClient(new ClientConfig().register(LoggingFeature.class)); + try { + final Response r = client.target(webServer.getServiceUri()).path("/status.html").request().get(); + Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode()); + } finally { + webServer.stop(); + } + log.info("-- Exiting {} test --", methodName); + } + @Test void testGetMetrics() throws Exception { log.info("-- Starting {} test --", methodName); From 153637d919d703dee4461d6a2b25e631de90156c Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 26 Oct 2023 09:29:32 +0800 Subject: [PATCH 338/494] [fix] [build] Fix in-correct license definetion (#21435) --- distribution/server/src/assemble/LICENSE.bin.txt | 3 +-- pulsar-sql/presto-distribution/LICENSE | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index fe5a3989b16f5..b1cf0cf6985a9 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -435,7 +435,6 @@ The Apache Software License, Version 2.0 - io.grpc-grpc-services-1.55.3.jar - io.grpc-grpc-xds-1.55.3.jar - io.grpc-grpc-rls-1.55.3.jar - - com.google.auto.service-auto-service-annotations-1.0.jar - io.grpc-grpc-servlet-1.55.3.jar - io.grpc-grpc-servlet-jakarta-1.55.3.jar * Perfmark @@ -479,7 +478,7 @@ The Apache Software License, Version 2.0 - io.vertx-vertx-core-4.3.8.jar - io.vertx-vertx-web-4.3.8.jar - io.vertx-vertx-web-common-4.3.8.jar - - io.vertx-vertx-grpc-4.3.8.jar + - io.vertx-vertx-grpc-4.3.5.jar * Apache ZooKeeper - org.apache.zookeeper-zookeeper-3.8.3.jar - org.apache.zookeeper-zookeeper-jute-3.8.3.jar diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index d26cd5f80f987..784143e96d6df 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -480,8 +480,6 @@ The Apache Software License, Version 2.0 - swagger-annotations-1.6.10.jar * Perfmark - perfmark-api-0.26.0.jar - * Annotations - - auto-service-annotations-1.0.jar * RabbitMQ Java Client - amqp-client-5.5.3.jar * Stream Lib @@ -524,9 +522,6 @@ MIT License - jcl-over-slf4j-1.7.32.jar * Checker Qual - checker-qual-3.33.0.jar - * Annotations - - animal-sniffer-annotations-1.21.jar - - annotations-4.1.1.4.jar * ScribeJava - scribejava-apis-6.9.0.jar - scribejava-core-6.9.0.jar From 79aedefc7f25eb0b460a5ef742ffd2e78123393c Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 26 Oct 2023 15:01:45 +0800 Subject: [PATCH 339/494] Release 3.0.2 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 132 files changed, 134 insertions(+), 134 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 5c4c62b1b8d8a..fdd3ebfcf0f0f 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1 + 3.0.2 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index cb2a36d6b5685..8c7a62dc27789 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1 + 3.0.2 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 9e2f46435ff58..16fc4ca2b6300 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1 + 3.0.2 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 9590ece2e0629..70568a144b18f 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 9fd1db11a503c..3fb0f91cf9b2b 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,12 +31,12 @@ org.apache.pulsar buildtools - 3.0.1 + 3.0.2 jar Pulsar Build Tools - 2023-07-13T02:30:03Z + 2023-10-26T07:01:13Z 1.8 1.8 3.1.0 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 8803f2d6c90ef..80af770a80277 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1 + 3.0.2 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index cae5539821e88..064e9838ad308 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1 + 3.0.2 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 5cf02f2f19fc3..9ab3897ec4f6b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 05988001502b4..ec518743b6966 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1 + 3.0.2 .. diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 8bdc59c562770..db2c2c9faddc8 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1 + 3.0.2 .. diff --git a/docker/pom.xml b/docker/pom.xml index 6d553e59f221a..3da8a9658dbe4 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index b27d0b081873b..e860f7d8d78e1 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.1 + 3.0.2 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 8761e57dbc340..c63767fc0b71b 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.1 + 3.0.2 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index ee2a64b1f320a..34504c0c3b448 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index a4659122188b2..cd70d81efe294 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 .. diff --git a/pom.xml b/pom.xml index 33176ee70635d..45b5988039671 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -92,7 +92,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2023-07-13T02:30:03Z + 2023-10-26T07:01:13Z true - conf/schema_example.conf + conf/schema_example.json **/templates/*.tpl diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java index 7e8f55429244b..f13a4dcfbdceb 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java @@ -402,7 +402,7 @@ public void testSchemaCLI() throws Exception { "upload", topicName, "-f", - "/pulsar/conf/schema_example.conf" + "/pulsar/conf/schema_example.json" ); result.assertNoOutput(); From cd5d2bef8d65c0f6158b8eb4b7ca7fbbde7028c1 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 27 Oct 2023 16:17:10 +0800 Subject: [PATCH 342/494] Release 3.0.2 --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 132 files changed, 134 insertions(+), 134 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 5c4c62b1b8d8a..fdd3ebfcf0f0f 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1 + 3.0.2 .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index cb2a36d6b5685..8c7a62dc27789 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1 + 3.0.2 .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 9e2f46435ff58..16fc4ca2b6300 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.1 + 3.0.2 .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 9590ece2e0629..70568a144b18f 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 9fd1db11a503c..959a092a2e64a 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,12 +31,12 @@ org.apache.pulsar buildtools - 3.0.1 + 3.0.2 jar Pulsar Build Tools - 2023-07-13T02:30:03Z + 2023-10-27T08:17:00Z 1.8 1.8 3.1.0 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 8803f2d6c90ef..80af770a80277 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1 + 3.0.2 .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index cae5539821e88..064e9838ad308 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1 + 3.0.2 .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 5cf02f2f19fc3..9ab3897ec4f6b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 05988001502b4..ec518743b6966 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1 + 3.0.2 .. diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 8bdc59c562770..db2c2c9faddc8 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.1 + 3.0.2 .. diff --git a/docker/pom.xml b/docker/pom.xml index 6d553e59f221a..3da8a9658dbe4 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index b27d0b081873b..e860f7d8d78e1 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.1 + 3.0.2 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 8761e57dbc340..c63767fc0b71b 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.1 + 3.0.2 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index ee2a64b1f320a..34504c0c3b448 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index a4659122188b2..cd70d81efe294 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 .. diff --git a/pom.xml b/pom.xml index f688fa55aca25..d27c9d3945d50 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 3.0.1 + 3.0.2 Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -92,7 +92,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2023-07-13T02:30:03Z + 2023-10-27T08:17:00Z true diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java index 726f5ae312d19..33034ddb3fe0f 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/TestZKServer.java @@ -19,39 +19,33 @@ package org.apache.pulsar.metadata; import static org.testng.Assert.assertTrue; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; -import java.net.InetSocketAddress; +import java.lang.reflect.Field; import java.net.Socket; - -import java.nio.charset.StandardCharsets; - +import java.util.Properties; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; - import org.apache.commons.io.FileUtils; -import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.server.ContainerManager; -import org.apache.zookeeper.server.NIOServerCnxnFactory; -import org.apache.zookeeper.server.Request; -import org.apache.zookeeper.server.RequestProcessor; import org.apache.zookeeper.server.ServerCnxnFactory; -import org.apache.zookeeper.server.SessionTracker; import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.ZooKeeperServerMain; +import org.apache.zookeeper.server.embedded.ExitHandler; +import org.apache.zookeeper.server.embedded.ZooKeeperServerEmbedded; import org.assertj.core.util.Files; @Slf4j public class TestZKServer implements AutoCloseable { + public static final int TICK_TIME = 1000; - protected ZooKeeperServer zks; - private final File zkDataDir; - private ServerCnxnFactory serverFactory; - private ContainerManager containerManager; - private int zkPort = 0; + private final File zkDataDir; + private int zkPort; // initially this is zero + private ZooKeeperServerEmbedded zooKeeperServerEmbedded; public TestZKServer() throws Exception { this.zkDataDir = Files.newTemporaryFolder(); @@ -64,86 +58,86 @@ public TestZKServer() throws Exception { } public void start() throws Exception { - this.zks = new ZooKeeperServer(zkDataDir, zkDataDir, TICK_TIME); - this.zks.setMaxSessionTimeout(300_000); - this.serverFactory = new NIOServerCnxnFactory(); - this.serverFactory.configure(new InetSocketAddress(zkPort), 1000); - this.serverFactory.startup(zks, true); - - this.zkPort = serverFactory.getLocalPort(); - log.info("Started test ZK server on port {}", zkPort); + final Properties configZookeeper = new Properties(); + configZookeeper.put("clientPort", zkPort + ""); + configZookeeper.put("host", "127.0.0.1"); + configZookeeper.put("ticktime", TICK_TIME + ""); + zooKeeperServerEmbedded = ZooKeeperServerEmbedded + .builder() + .baseDir(zkDataDir.toPath()) + .configuration(configZookeeper) + .exitHandler(ExitHandler.LOG_ONLY) + .build(); + + zooKeeperServerEmbedded.start(60_000); + log.info("Started test ZK server on at {}", zooKeeperServerEmbedded.getConnectionString()); + + ZooKeeperServerMain zooKeeperServerMain = getZooKeeperServerMain(zooKeeperServerEmbedded); + ServerCnxnFactory serverCnxnFactory = getServerCnxnFactory(zooKeeperServerMain); + // save the port, in order to allow restarting on the same port + zkPort = serverCnxnFactory.getLocalPort(); boolean zkServerReady = waitForServerUp(this.getConnectionString(), 30_000); assertTrue(zkServerReady); + } - this.containerManager = new ContainerManager(zks.getZKDatabase(), new RequestProcessor() { - @Override - public void processRequest(Request request) throws RequestProcessorException { - String path = StandardCharsets.UTF_8.decode(request.request).toString(); - try { - zks.getZKDatabase().getDataTree().deleteNode(path, -1); - } catch (KeeperException.NoNodeException e) { - // Ok - } - } + @SneakyThrows + private static ZooKeeperServerMain getZooKeeperServerMain(ZooKeeperServerEmbedded zooKeeperServerEmbedded) { + ZooKeeperServerMain zooKeeperServerMain = readField(zooKeeperServerEmbedded.getClass(), + "mainsingle", zooKeeperServerEmbedded); + return zooKeeperServerMain; + } - @Override - public void shutdown() { + @SneakyThrows + private static ContainerManager getContainerManager(ZooKeeperServerMain zooKeeperServerMain) { + ContainerManager containerManager = readField(ZooKeeperServerMain.class, "containerManager", zooKeeperServerMain); + return containerManager; + } - } - }, 10, 10000, 0L); + @SneakyThrows + private static ZooKeeperServer getZooKeeperServer(ZooKeeperServerMain zooKeeperServerMain) { + ServerCnxnFactory serverCnxnFactory = getServerCnxnFactory(zooKeeperServerMain); + ZooKeeperServer zkServer = readField(ServerCnxnFactory.class, "zkServer", serverCnxnFactory); + return zkServer; + } + + @SneakyThrows + private static T readField(Class clazz, String field, Object object) { + Field declaredField = clazz.getDeclaredField(field); + boolean accessible = declaredField.isAccessible(); + if (!accessible) { + declaredField.setAccessible(true); + } + try { + return (T) declaredField.get(object); + } finally { + declaredField.setAccessible(accessible); + } + } + + private static ServerCnxnFactory getServerCnxnFactory(ZooKeeperServerMain zooKeeperServerMain) throws Exception { + ServerCnxnFactory serverCnxnFactory = readField(ZooKeeperServerMain.class, "cnxnFactory", zooKeeperServerMain); + return serverCnxnFactory; } public void checkContainers() throws Exception { // Make sure the container nodes are actually deleted Thread.sleep(1000); + ContainerManager containerManager = getContainerManager(getZooKeeperServerMain(zooKeeperServerEmbedded)); containerManager.checkContainers(); } public void stop() throws Exception { - if (containerManager != null) { - containerManager.stop(); - containerManager = null; - } - - if (serverFactory != null) { - serverFactory.shutdown(); - serverFactory = null; - } - - if (zks != null) { - SessionTracker sessionTracker = zks.getSessionTracker(); - zks.shutdown(); - zks.getZKDatabase().close(); - if (sessionTracker instanceof Thread) { - Thread sessionTrackerThread = (Thread) sessionTracker; - sessionTrackerThread.interrupt(); - sessionTrackerThread.join(); - } - zks = null; + if (zooKeeperServerEmbedded != null) { + zooKeeperServerEmbedded.close(); } - log.info("Stopped test ZK server"); } public void expireSession(long sessionId) { - zks.expire(new SessionTracker.Session() { - @Override - public long getSessionId() { - return sessionId; - } - - @Override - public int getTimeout() { - return 10_000; - } - - @Override - public boolean isClosing() { - return false; - } - }); + getZooKeeperServer(getZooKeeperServerMain(zooKeeperServerEmbedded)) + .expire(sessionId); } @Override @@ -152,12 +146,9 @@ public void close() throws Exception { FileUtils.deleteDirectory(zkDataDir); } - public int getPort() { - return zkPort; - } - + @SneakyThrows public String getConnectionString() { - return "127.0.0.1:" + getPort(); + return zooKeeperServerEmbedded.getConnectionString(); } public static boolean waitForServerUp(String hp, long timeout) { diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index ccb0c4d1c0d3c..4eebb815cf0f3 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -71,6 +71,12 @@ + + org.hamcrest + hamcrest + test + + io.dropwizard.metrics diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index e16728f3ff884..29bde2dada25b 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -472,8 +472,8 @@ The Apache Software License, Version 2.0 - memory-0.8.3.jar - sketches-core-0.8.3.jar * Apache Zookeeper - - zookeeper-3.8.3.jar - - zookeeper-jute-3.8.3.jar + - zookeeper-3.9.1.jar + - zookeeper-jute-3.9.1.jar * Apache Yetus Audience Annotations - audience-annotations-0.12.0.jar * Swagger From 0f7434580f285d027a625d9de7ca41e3e74a6365 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 23 Jun 2023 15:18:35 +0300 Subject: [PATCH 374/494] [fix][sec] Upgrade Bouncycastle to 1.75 to address CVE-2023-33201 (#20631) --- bouncy-castle/bc/LICENSE | 6 +-- bouncy-castle/bc/pom.xml | 4 +- distribution/server/pom.xml | 4 ++ .../server/src/assemble/LICENSE.bin.txt | 8 ++-- .../shell/src/assemble/LICENSE.bin.txt | 8 ++-- pom.xml | 46 +++++++++++++++++-- pulsar-broker-auth-athenz/pom.xml | 5 ++ pulsar-client-auth-athenz/pom.xml | 5 ++ .../client/impl/crypto/MessageCryptoBc.java | 32 ++++++++++--- pulsar-io/aerospike/pom.xml | 10 ++++ pulsar-sql/presto-distribution/LICENSE | 8 ++-- tests/integration/pom.xml | 5 ++ tiered-storage/file-system/pom.xml | 18 ++++++-- 13 files changed, 127 insertions(+), 32 deletions(-) diff --git a/bouncy-castle/bc/LICENSE b/bouncy-castle/bc/LICENSE index 5921755346e9e..dae8f16df5b82 100644 --- a/bouncy-castle/bc/LICENSE +++ b/bouncy-castle/bc/LICENSE @@ -205,6 +205,6 @@ This projects includes binary packages with the following licenses: Bouncy Castle License * Bouncy Castle -- licenses/LICENSE-bouncycastle.txt - - org.bouncycastle-bcpkix-jdk15on-1.60.jar - - org.bouncycastle-bcprov-jdk15on-1.60.jar - - org.bouncycastle-bcprov-ext-jdk15on-1.60.jar + - org.bouncycastle-bcpkix-jdk18on-1.75.jar + - org.bouncycastle-bcprov-jdk18on-1.75.jar + - org.bouncycastle-bcprov-ext-jdk18on-1.75.jar diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index fdd3ebfcf0f0f..8a26082b126a1 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -42,13 +42,13 @@ org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${bouncycastle.version} org.bouncycastle - bcprov-ext-jdk15on + bcprov-ext-jdk18on ${bouncycastle.version} diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 7ee2bd88128af..0e033b10ed44e 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -259,6 +259,10 @@ io.grpc grpc-all + + org.bouncycastle + bcpkix-jdk18on + io.perfmark diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index c53da3bd53618..8dc361cab3af4 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -575,10 +575,10 @@ Creative Commons Attribution License Bouncy Castle License * Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt - - org.bouncycastle-bcpkix-jdk15on-1.69.jar - - org.bouncycastle-bcprov-ext-jdk15on-1.69.jar - - org.bouncycastle-bcprov-jdk15on-1.69.jar - - org.bouncycastle-bcutil-jdk15on-1.69.jar + - org.bouncycastle-bcpkix-jdk18on-1.75.jar + - org.bouncycastle-bcprov-ext-jdk18on-1.75.jar + - org.bouncycastle-bcprov-jdk18on-1.75.jar + - org.bouncycastle-bcutil-jdk18on-1.75.jar ------------------------ diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index e69b74e7a9ebf..43bbfdec5bff8 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -470,10 +470,10 @@ Creative Commons Attribution License Bouncy Castle License * Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt - - bcpkix-jdk15on-1.69.jar - - bcprov-ext-jdk15on-1.69.jar - - bcprov-jdk15on-1.69.jar - - bcutil-jdk15on-1.69.jar + - bcpkix-jdk18on-1.75.jar + - bcprov-ext-jdk18on-1.75.jar + - bcprov-jdk18on-1.75.jar + - bcutil-jdk18on-1.75.jar ------------------------ diff --git a/pom.xml b/pom.xml index 0389eeed2ad58..24b712bdadb8f 100644 --- a/pom.xml +++ b/pom.xml @@ -152,7 +152,7 @@ flexible messaging model and an intuitive client API. 1.7.32 4.4 2.18.0 - 1.69 + 1.75 1.0.6 1.0.2.3 2.14.2 @@ -824,9 +824,15 @@ flexible messaging model and an intuitive client API. - com.github.docker-java - docker-java-core - ${docker-java.version} + com.github.docker-java + docker-java-core + ${docker-java.version} + + + org.bouncycastle + * + + com.github.docker-java @@ -892,7 +898,7 @@ flexible messaging model and an intuitive client API. org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${bouncycastle.version} @@ -924,6 +930,24 @@ flexible messaging model and an intuitive client API. com.yahoo.athenz athenz-cert-refresher ${athenz.version} + + + org.bouncycastle + * + + + + + + com.yahoo.athenz + athenz-auth-core + ${athenz.version} + + + org.bouncycastle + * + + @@ -1068,6 +1092,18 @@ flexible messaging model and an intuitive client API. + + io.grpc + grpc-xds + ${grpc.version} + + + org.bouncycastle + * + + + + com.google.http-client google-http-client diff --git a/pulsar-broker-auth-athenz/pom.xml b/pulsar-broker-auth-athenz/pom.xml index b8837ce67fcff..6711a60bc89bc 100644 --- a/pulsar-broker-auth-athenz/pom.xml +++ b/pulsar-broker-auth-athenz/pom.xml @@ -53,6 +53,11 @@ athenz-zpe-java-client + + org.bouncycastle + bcpkix-jdk18on + + diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index 2149cfb2a2fe3..81315611e9bef 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -52,6 +52,11 @@ athenz-cert-refresher + + org.bouncycastle + bcpkix-jdk18on + + com.google.guava guava diff --git a/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java b/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java index 2d7b779fa7b6c..146f066ae2c6b 100644 --- a/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java +++ b/pulsar-client-messagecrypto-bc/src/main/java/org/apache/pulsar/client/impl/crypto/MessageCryptoBc.java @@ -35,6 +35,7 @@ import java.security.PublicKey; import java.security.SecureRandom; import java.security.Security; +import java.security.spec.AlgorithmParameterSpec; import java.security.spec.InvalidKeySpecException; import java.util.HashMap; import java.util.List; @@ -73,6 +74,7 @@ import org.bouncycastle.jce.spec.ECParameterSpec; import org.bouncycastle.jce.spec.ECPrivateKeySpec; import org.bouncycastle.jce.spec.ECPublicKeySpec; +import org.bouncycastle.jce.spec.IESParameterSpec; import org.bouncycastle.openssl.PEMException; import org.bouncycastle.openssl.PEMKeyPair; import org.bouncycastle.openssl.PEMParser; @@ -172,6 +174,7 @@ public SecretKey load(ByteBuffer key) { dataKey = keyGenerator.generateKey(); iv = new byte[IV_LEN]; + } private PublicKey loadPublicKey(byte[] keyBytes) throws Exception { @@ -322,22 +325,27 @@ private void addPublicKeyCipher(String keyName, CryptoKeyReader keyReader) throw byte[] encryptedKey; try { - + AlgorithmParameterSpec params = null; // Encrypt data key using public key if (RSA.equals(pubKey.getAlgorithm())) { dataKeyCipher = Cipher.getInstance(RSA_TRANS, BouncyCastleProvider.PROVIDER_NAME); } else if (ECDSA.equals(pubKey.getAlgorithm())) { dataKeyCipher = Cipher.getInstance(ECIES, BouncyCastleProvider.PROVIDER_NAME); + params = createIESParameterSpec(); } else { String msg = logCtx + "Unsupported key type " + pubKey.getAlgorithm() + " for key " + keyName; log.error(msg); throw new PulsarClientException.CryptoException(msg); } - dataKeyCipher.init(Cipher.ENCRYPT_MODE, pubKey); + if (params != null) { + dataKeyCipher.init(Cipher.ENCRYPT_MODE, pubKey, params); + } else { + dataKeyCipher.init(Cipher.ENCRYPT_MODE, pubKey); + } encryptedKey = dataKeyCipher.doFinal(dataKey.getEncoded()); } catch (IllegalBlockSizeException | BadPaddingException | NoSuchAlgorithmException | NoSuchProviderException - | NoSuchPaddingException | InvalidKeyException e) { + | NoSuchPaddingException | InvalidKeyException | InvalidAlgorithmParameterException e) { log.error("{} Failed to encrypt data key {}. {}", logCtx, keyName, e.getMessage()); throw new PulsarClientException.CryptoException(e.getMessage()); } @@ -345,6 +353,13 @@ private void addPublicKeyCipher(String keyName, CryptoKeyReader keyReader) throw encryptedDataKeyMap.put(keyName, eki); } + // required since Bouncycastle 1.72 when using ECIES, it is required to pass in an IESParameterSpec + private IESParameterSpec createIESParameterSpec() { + // the IESParameterSpec to use was discovered by debugging BouncyCastle 1.69 and running the + // test org.apache.pulsar.client.api.SimpleProducerConsumerTest#testCryptoWithChunking + return new IESParameterSpec(null, null, 128); + } + /* * Remove a key

Remove the key identified by the keyName from the list of keys.

* @@ -474,23 +489,28 @@ private boolean decryptDataKey(String keyName, byte[] encryptedDataKey, Listcom.aerospike aerospike-client-bc ${aerospike-client.version} + + + org.bouncycastle + * + + + + + org.bouncycastle + bcpkix-jdk18on diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 29bde2dada25b..8f67f2f7ef479 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -590,7 +590,7 @@ Creative Commons Attribution License Bouncy Castle License * Bouncy Castle -- licenses/LICENSE-bouncycastle.txt - - bcpkix-jdk15on-1.69.jar - - bcprov-ext-jdk15on-1.69.jar - - bcprov-jdk15on-1.69.jar - - bcutil-jdk15on-1.69.jar + - bcpkix-jdk18on-1.75.jar + - bcprov-ext-jdk18on-1.75.jar + - bcprov-jdk18on-1.75.jar + - bcutil-jdk18on-1.75.jar diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index c5acf24ab43f8..3600ead12f11d 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -126,6 +126,11 @@ docker-java-core test + + org.bouncycastle + bcpkix-jdk18on + test + org.apache.pulsar diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index cf482f43a4e1c..5b4474c977b66 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -110,13 +110,23 @@ ${hdfs-offload-version3} test - - io.netty - netty-all - + + io.netty + netty-all + + + org.bouncycastle + * + + + org.bouncycastle + bcpkix-jdk18on + test + + io.netty netty-codec-http From f87e657bc0b7bb1a845dbd151a92bd3164ff3b9a Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 1 Dec 2023 13:58:51 +0800 Subject: [PATCH 375/494] Set project version 3.0.3-SNAPSHOT --- bouncy-castle/bc/pom.xml | 2 +- bouncy-castle/bcfips-include-test/pom.xml | 2 +- bouncy-castle/bcfips/pom.xml | 2 +- bouncy-castle/pom.xml | 2 +- buildtools/pom.xml | 4 ++-- distribution/io/pom.xml | 2 +- distribution/offloaders/pom.xml | 2 +- distribution/pom.xml | 2 +- distribution/server/pom.xml | 2 +- distribution/shell/pom.xml | 2 +- docker/pom.xml | 2 +- docker/pulsar-all/pom.xml | 2 +- docker/pulsar/pom.xml | 2 +- jclouds-shaded/pom.xml | 2 +- managed-ledger/pom.xml | 2 +- pom.xml | 4 ++-- pulsar-broker-auth-athenz/pom.xml | 2 +- pulsar-broker-auth-oidc/pom.xml | 2 +- pulsar-broker-auth-sasl/pom.xml | 2 +- pulsar-broker-common/pom.xml | 2 +- pulsar-broker/pom.xml | 2 +- pulsar-client-1x-base/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-1x/pom.xml | 2 +- pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml | 2 +- pulsar-client-admin-api/pom.xml | 2 +- pulsar-client-admin-shaded/pom.xml | 2 +- pulsar-client-admin/pom.xml | 2 +- pulsar-client-all/pom.xml | 2 +- pulsar-client-api/pom.xml | 2 +- pulsar-client-auth-athenz/pom.xml | 2 +- pulsar-client-auth-sasl/pom.xml | 2 +- pulsar-client-messagecrypto-bc/pom.xml | 2 +- pulsar-client-shaded/pom.xml | 2 +- pulsar-client-tools-api/pom.xml | 2 +- pulsar-client-tools-customcommand-example/pom.xml | 2 +- pulsar-client-tools-test/pom.xml | 2 +- pulsar-client-tools/pom.xml | 2 +- pulsar-client/pom.xml | 2 +- pulsar-common/pom.xml | 2 +- pulsar-config-validation/pom.xml | 2 +- pulsar-functions/api-java/pom.xml | 2 +- pulsar-functions/instance/pom.xml | 2 +- pulsar-functions/java-examples-builtin/pom.xml | 2 +- pulsar-functions/java-examples/pom.xml | 2 +- pulsar-functions/localrun-shaded/pom.xml | 2 +- pulsar-functions/localrun/pom.xml | 2 +- pulsar-functions/pom.xml | 2 +- pulsar-functions/proto/pom.xml | 2 +- pulsar-functions/runtime-all/pom.xml | 2 +- pulsar-functions/runtime/pom.xml | 2 +- pulsar-functions/secrets/pom.xml | 2 +- pulsar-functions/utils/pom.xml | 2 +- pulsar-functions/worker/pom.xml | 2 +- pulsar-io/aerospike/pom.xml | 2 +- pulsar-io/alluxio/pom.xml | 2 +- pulsar-io/aws/pom.xml | 2 +- pulsar-io/batch-data-generator/pom.xml | 2 +- pulsar-io/batch-discovery-triggerers/pom.xml | 2 +- pulsar-io/canal/pom.xml | 2 +- pulsar-io/cassandra/pom.xml | 2 +- pulsar-io/common/pom.xml | 2 +- pulsar-io/core/pom.xml | 2 +- pulsar-io/data-generator/pom.xml | 2 +- pulsar-io/debezium/core/pom.xml | 2 +- pulsar-io/debezium/mongodb/pom.xml | 2 +- pulsar-io/debezium/mssql/pom.xml | 2 +- pulsar-io/debezium/mysql/pom.xml | 2 +- pulsar-io/debezium/oracle/pom.xml | 2 +- pulsar-io/debezium/pom.xml | 2 +- pulsar-io/debezium/postgres/pom.xml | 2 +- pulsar-io/docs/pom.xml | 2 +- pulsar-io/dynamodb/pom.xml | 2 +- pulsar-io/elastic-search/pom.xml | 2 +- pulsar-io/file/pom.xml | 2 +- pulsar-io/flume/pom.xml | 2 +- pulsar-io/hbase/pom.xml | 2 +- pulsar-io/hdfs2/pom.xml | 2 +- pulsar-io/hdfs3/pom.xml | 2 +- pulsar-io/http/pom.xml | 2 +- pulsar-io/influxdb/pom.xml | 2 +- pulsar-io/jdbc/clickhouse/pom.xml | 2 +- pulsar-io/jdbc/core/pom.xml | 2 +- pulsar-io/jdbc/mariadb/pom.xml | 2 +- pulsar-io/jdbc/openmldb/pom.xml | 2 +- pulsar-io/jdbc/pom.xml | 2 +- pulsar-io/jdbc/postgres/pom.xml | 2 +- pulsar-io/jdbc/sqlite/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 2 +- pulsar-io/kafka-connect-adaptor/pom.xml | 2 +- pulsar-io/kafka/pom.xml | 2 +- pulsar-io/kinesis/pom.xml | 2 +- pulsar-io/mongo/pom.xml | 2 +- pulsar-io/netty/pom.xml | 2 +- pulsar-io/nsq/pom.xml | 2 +- pulsar-io/pom.xml | 2 +- pulsar-io/rabbitmq/pom.xml | 2 +- pulsar-io/redis/pom.xml | 2 +- pulsar-io/solr/pom.xml | 2 +- pulsar-io/twitter/pom.xml | 2 +- pulsar-metadata/pom.xml | 2 +- pulsar-package-management/bookkeeper-storage/pom.xml | 2 +- pulsar-package-management/core/pom.xml | 2 +- pulsar-package-management/filesystem-storage/pom.xml | 2 +- pulsar-package-management/pom.xml | 2 +- pulsar-proxy/pom.xml | 2 +- pulsar-sql/pom.xml | 2 +- pulsar-sql/presto-distribution/pom.xml | 2 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 2 +- pulsar-sql/presto-pulsar/pom.xml | 2 +- pulsar-testclient/pom.xml | 2 +- pulsar-transaction/common/pom.xml | 2 +- pulsar-transaction/coordinator/pom.xml | 2 +- pulsar-transaction/pom.xml | 2 +- pulsar-websocket/pom.xml | 2 +- structured-event-log/pom.xml | 2 +- testmocks/pom.xml | 2 +- tests/bc_2_0_0/pom.xml | 2 +- tests/bc_2_0_1/pom.xml | 2 +- tests/bc_2_6_0/pom.xml | 2 +- tests/docker-images/java-test-functions/pom.xml | 2 +- tests/docker-images/java-test-image/pom.xml | 2 +- tests/docker-images/java-test-plugins/pom.xml | 2 +- tests/docker-images/latest-version-image/pom.xml | 2 +- tests/docker-images/pom.xml | 2 +- tests/integration/pom.xml | 2 +- tests/pom.xml | 2 +- tests/pulsar-client-admin-shade-test/pom.xml | 2 +- tests/pulsar-client-all-shade-test/pom.xml | 2 +- tests/pulsar-client-shade-test/pom.xml | 2 +- tiered-storage/file-system/pom.xml | 2 +- tiered-storage/jcloud/pom.xml | 2 +- tiered-storage/pom.xml | 2 +- 132 files changed, 134 insertions(+), 134 deletions(-) diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index 8a26082b126a1..a52fd8cec9b80 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 8c7a62dc27789..2770b5127c860 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -24,7 +24,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 16fc4ca2b6300..3fb9af4eb040d 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar bouncy-castle-parent - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index 70568a144b18f..c8e5ff2f550b8 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 7226796ea92d4..4845ec7439bf8 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -31,12 +31,12 @@ org.apache.pulsar buildtools - 3.0.2 + 3.0.3-SNAPSHOT jar Pulsar Build Tools - 2023-11-17T07:50:55Z + 2023-12-01T05:56:29Z 1.8 1.8 3.1.0 diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 80af770a80277..5f943d6f328d8 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 064e9838ad308..ebc069838e09e 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/distribution/pom.xml b/distribution/pom.xml index 9ab3897ec4f6b..08eabeec53d11 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 0e033b10ed44e..b51bae3177750 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index db2c2c9faddc8..5a1df2aec4421 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar distribution - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/docker/pom.xml b/docker/pom.xml index 3da8a9658dbe4..8ff2a8ae7ebba 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.2 + 3.0.3-SNAPSHOT docker-images Apache Pulsar :: Docker Images diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index e860f7d8d78e1..04f01272b7afb 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.2 + 3.0.3-SNAPSHOT 4.0.0 pulsar-all-docker-image diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index c63767fc0b71b..413cb3cd0671f 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -23,7 +23,7 @@ org.apache.pulsar docker-images - 3.0.2 + 3.0.3-SNAPSHOT 4.0.0 pulsar-docker-image diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 34504c0c3b448..e9a63d5d686e5 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -26,7 +26,7 @@ org.apache.pulsar pulsar - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 89a5f23823973..8a1078fcc2e27 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -25,7 +25,7 @@ org.apache.pulsar pulsar - 3.0.2 + 3.0.3-SNAPSHOT .. diff --git a/pom.xml b/pom.xml index 24b712bdadb8f..fcafa97e98d11 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ org.apache.pulsar pulsar - 3.0.2 + 3.0.3-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very @@ -92,7 +92,7 @@ flexible messaging model and an intuitive client API. UTF-8 UTF-8 - 2023-11-17T07:50:55Z + 2023-12-01T05:56:29Z true 1.21 - 4.16.3 + 4.16.4 3.9.1 1.5.0 1.10.0 diff --git a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java index ec5f77f79464b..970c45964f784 100644 --- a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java +++ b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java @@ -42,6 +42,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import lombok.Cleanup; import org.apache.bookkeeper.bookie.BookieImpl; @@ -425,8 +426,16 @@ public void testInnerDelayedAuditOfLostBookies() throws Exception { // wait for 5 seconds before starting the recovery work when a bookie fails urLedgerMgr.setLostBookieRecoveryDelay(5); - // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -442,9 +451,10 @@ public void testInnerDelayedAuditOfLostBookies() throws Exception { urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + "is not listed in the ledger as missing replica :" + data, - data.contains(shutdownBookie)); + data.contains(shutdownBookieRef.get())); } /** @@ -503,7 +513,16 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws urLedgerMgr.setLostBookieRecoveryDelay(50); // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -522,9 +541,10 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartImmediately() throws urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + "is not listed in the ledger as missing replica :" + data, - data.contains(shutdownBookie)); + data.contains(shutdownBookieRef.get())); } @Test @@ -547,7 +567,16 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Excep urLedgerMgr.setLostBookieRecoveryDelay(3); // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + AtomicReference shutdownBookieRef = new AtomicReference<>(); + CountDownLatch shutdownLatch = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie = shutDownNonAuditorBookie(); + shutdownBookieRef.set(shutdownBookie); + shutdownLatch.countDown(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -573,9 +602,10 @@ public void testRescheduleOfDelayedAuditOfLostBookiesToStartLater() throws Excep urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie + shutdownLatch.await(); + assertTrue("Bookie " + shutdownBookieRef.get() + "is not listed in the ledger as missing replica :" + data, - data.contains(shutdownBookie)); + data.contains(shutdownBookieRef.get())); } @Test @@ -664,7 +694,12 @@ public void testTriggerAuditorWithPendingAuditTask() throws Exception { urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + new Thread(() -> { + try { + shutDownNonAuditorBookie(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -715,7 +750,12 @@ public void testTriggerAuditorBySettingDelayToZeroWithPendingAuditTask() throws urLedgerMgr.setLostBookieRecoveryDelay(lostBookieRecoveryDelay); // shutdown a non auditor bookie; choosing non-auditor to avoid another election - String shutdownBookie = shutDownNonAuditorBookie(); + new Thread(() -> { + try { + shutDownNonAuditorBookie(); + } catch (Exception ignore) { + } + }).start(); if (LOG.isDebugEnabled()) { LOG.debug("Waiting for ledgers to be marked as under replicated"); @@ -767,8 +807,17 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception { // wait for 10 seconds before starting the recovery work when a bookie fails urLedgerMgr.setLostBookieRecoveryDelay(10); - // shutdown a non auditor bookie to avoid an election - String shutdownBookie1 = shutDownNonAuditorBookie(); + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + AtomicReference shutdownBookieRef1 = new AtomicReference<>(); + CountDownLatch shutdownLatch1 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie1 = shutDownNonAuditorBookie(); + shutdownBookieRef1.set(shutdownBookie1); + shutdownLatch1.countDown(); + } catch (Exception ignore) { + } + }).start(); // wait for 3 seconds and there shouldn't be any under replicated ledgers // because we have delayed the start of audit by 10 seconds @@ -780,7 +829,16 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception { // the history about having delayed recovery remains. Hence we make sure // we bring down a non auditor bookie. This should cause the audit to take // place immediately and not wait for the remaining 7 seconds to elapse - String shutdownBookie2 = shutDownNonAuditorBookie(); + AtomicReference shutdownBookieRef2 = new AtomicReference<>(); + CountDownLatch shutdownLatch2 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie2 = shutDownNonAuditorBookie(); + shutdownBookieRef2.set(shutdownBookie2); + shutdownLatch2.countDown(); + } catch (Exception ignore) { + } + }).start(); // 2 second grace period for the ledgers to get reported as under replicated Thread.sleep(2000); @@ -793,9 +851,11 @@ public void testDelayedAuditWithMultipleBookieFailures() throws Exception { urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie1 + shutdownBookie2 + shutdownLatch1.await(); + shutdownLatch2.await(); + assertTrue("Bookie " + shutdownBookieRef1.get() + shutdownBookieRef2.get() + " are not listed in the ledger as missing replicas :" + data, - data.contains(shutdownBookie1) && data.contains(shutdownBookie2)); + data.contains(shutdownBookieRef1.get()) && data.contains(shutdownBookieRef2.get())); } /** @@ -825,7 +885,16 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception { // shutdown a non auditor bookie to avoid an election int idx1 = getShutDownNonAuditorBookieIdx(""); ServerConfiguration conf1 = confByIndex(idx1); - String shutdownBookie1 = shutdownBookie(idx1); + AtomicReference shutdownBookieRef1 = new AtomicReference<>(); + CountDownLatch shutdownLatch1 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie1 = shutdownBookie(idx1); + shutdownBookieRef1.set(shutdownBookie1); + shutdownLatch1.countDown(); + } catch (Exception ignore) { + } + }).start(); // wait for 2 seconds and there shouldn't be any under replicated ledgers // because we have delayed the start of audit by 5 seconds @@ -838,7 +907,17 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception { // Now to simulate the rolling upgrade, bring down a bookie different from // the one we brought down/up above. - String shutdownBookie2 = shutDownNonAuditorBookie(shutdownBookie1); + // shutdown a non auditor bookie; choosing non-auditor to avoid another election + AtomicReference shutdownBookieRef2 = new AtomicReference<>(); + CountDownLatch shutdownLatch2 = new CountDownLatch(1); + new Thread(() -> { + try { + String shutdownBookie2 = shutDownNonAuditorBookie(); + shutdownBookieRef2.set(shutdownBookie2); + shutdownLatch2.countDown(); + } catch (Exception ignore) { + } + }).start(); // since the first bookie that was brought down/up has come up, there is only // one bookie down at this time. Hence the lost bookie check shouldn't start @@ -856,11 +935,13 @@ public void testDelayedAuditWithRollingUpgrade() throws Exception { urLedgerList.contains(ledgerId)); Map urLedgerData = getUrLedgerData(urLedgerList); String data = urLedgerData.get(ledgerId); - assertTrue("Bookie " + shutdownBookie1 + "wrongly listed as missing the ledger: " + data, - !data.contains(shutdownBookie1)); - assertTrue("Bookie " + shutdownBookie2 + shutdownLatch1.await(); + shutdownLatch2.await(); + assertTrue("Bookie " + shutdownBookieRef1.get() + "wrongly listed as missing the ledger: " + data, + !data.contains(shutdownBookieRef1.get())); + assertTrue("Bookie " + shutdownBookieRef2.get() + " is not listed in the ledger as missing replicas :" + data, - data.contains(shutdownBookie2)); + data.contains(shutdownBookieRef2.get())); LOG.info("*****************Test Complete"); } From 503f85f6b312448ac6d1fd96b0bd2dda29070927 Mon Sep 17 00:00:00 2001 From: Hang Chen Date: Wed, 31 Jan 2024 11:17:59 +0800 Subject: [PATCH 436/494] [fix] [bk] Fix the BookKeeper license (#22000) --- pulsar-sql/presto-distribution/LICENSE | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 8f67f2f7ef479..398562c7dc22f 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -430,21 +430,21 @@ The Apache Software License, Version 2.0 - async-http-client-2.12.1.jar - async-http-client-netty-utils-2.12.1.jar * Apache Bookkeeper - - bookkeeper-common-4.16.3.jar - - bookkeeper-common-allocator-4.16.3.jar - - bookkeeper-proto-4.16.3.jar - - bookkeeper-server-4.16.3.jar - - bookkeeper-stats-api-4.16.3.jar - - bookkeeper-tools-framework-4.16.3.jar - - circe-checksum-4.16.3.jar - - codahale-metrics-provider-4.16.3.jar - - cpu-affinity-4.16.3.jar - - http-server-4.16.3.jar - - prometheus-metrics-provider-4.16.3.jar - - codahale-metrics-provider-4.16.3.jar - - bookkeeper-slogger-api-4.16.3.jar - - bookkeeper-slogger-slf4j-4.16.3.jar - - native-io-4.16.3.jar + - bookkeeper-common-4.16.4.jar + - bookkeeper-common-allocator-4.16.4.jar + - bookkeeper-proto-4.16.4.jar + - bookkeeper-server-4.16.4.jar + - bookkeeper-stats-api-4.16.4.jar + - bookkeeper-tools-framework-4.16.4.jar + - circe-checksum-4.16.4.jar + - codahale-metrics-provider-4.16.4.jar + - cpu-affinity-4.16.4.jar + - http-server-4.16.4.jar + - prometheus-metrics-provider-4.16.4.jar + - codahale-metrics-provider-4.16.4.jar + - bookkeeper-slogger-api-4.16.4.jar + - bookkeeper-slogger-slf4j-4.16.4.jar + - native-io-4.16.4.jar * Apache Commons - commons-cli-1.5.0.jar - commons-codec-1.15.jar From 43864019e3b8754c00960223cae3555a50774835 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Wed, 31 Jan 2024 16:19:11 +0800 Subject: [PATCH 437/494] [fix] [broker] [branch-3.0] Fast fix infinite HTTP call createSubscriptions caused by wrong topicName (#21997) Similar to: #20131 The master branch has fixed the issue by https://github.com/apache/pulsar/pull/19841( Since it will makes users can not receive the messages which created in mistake, we did not cherry-pick https://github.com/apache/pulsar/pull/19841 into other branches, see detail https://github.com/apache/pulsar/pull/19841) ### Motivation #### Background of Admin API `PersistentTopics.createSubscription` It works like this: 1. createSubscription( `tp1` ) 2. is partitioned topic? `no`: return subscriptions `yes`: createSubscription(`tp1-partition-0`)....createSubscription(`tp1-partition-n`) --- #### Background of the issue of `TopicName.getPartition(int index)` ```java String partitionedTopic = "tp1-partition-0-DLQ"; TopicName partition0 = partitionedTopic.getPartition(0);// Highlight: the partition0.toString() will be "tp1-partition-0-DLQ"(it is wrong).The correct value is "tp1-partition-0-DLQ-partition-0" ``` #### Issue Therefore, if there has a partitioned topic named `tp1-partition-0-DLQ`, the method `PersistentTopics.createSubscription` will works like this: 1. call Admin API ``PersistentTopics.createSubscription("tp1-partition-0-DLQ")` 2. is partitioned topic? 3. yes, call `TopicName.getPartition(0)` to get partition 0 and will get `tp1-partition-0-DLQ` , then loop to step-1. Then the infinite HTTP call `PersistentTopics.createSubscription` makes the broker crash. ### Modifications #### Quick fix(this PR does it) If hits the issue which makes the topic name wrong, do not loop to step 1. #### Long-term fix The PR https://github.com/apache/pulsar/pull/19841 fixes the issue which makes the topic name wrong, and this PR will create unfriendly compatibility, and PIP 263 https://github.com/apache/pulsar/issues/20033 will make compatibility good. --- .../pulsar/broker/admin/impl/PersistentTopicsBase.java | 2 +- ...TopicNameForInfiniteHttpCallGetSubscriptionsTest.java | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 2b9a34b382e3f..53c412bcf1d6b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -2349,7 +2349,7 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su .thenCompose(allowAutoTopicCreation -> getPartitionedTopicMetadataAsync(topicName, authoritative, allowAutoTopicCreation).thenAccept(partitionMetadata -> { final int numPartitions = partitionMetadata.partitions; - if (numPartitions > 0) { + if (partitionMetadata.partitions > 0 && !isUnexpectedTopicName(partitionMetadata)) { final CompletableFuture future = new CompletableFuture<>(); final AtomicInteger count = new AtomicInteger(numPartitions); final AtomicInteger failureCount = new AtomicInteger(0); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java index 2efc4f4e7803f..96f57a59bda13 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TopicNameForInfiniteHttpCallGetSubscriptionsTest.java @@ -53,7 +53,7 @@ protected void cleanup() throws Exception { } @Test - public void testInfiniteHttpCallGetSubscriptions() throws Exception { + public void testInfiniteHttpCallGetOrCreateSubscriptions() throws Exception { final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); final String partitionedTopicName = "persistent://my-property/my-ns/tp1_" + randomStr; final String topic_p0 = partitionedTopicName + TopicName.PARTITIONED_TOPIC_SUFFIX + "0"; @@ -65,6 +65,7 @@ public void testInfiniteHttpCallGetSubscriptions() throws Exception { // Do test. ProducerAndConsumerEntry pcEntry = triggerDLQCreated(topic_p0, topicDLQ, subscriptionName); admin.topics().getSubscriptions(topicDLQ); + admin.topics().createSubscription(topicDLQ, "s1", MessageId.earliest); // cleanup. pcEntry.consumer.close(); @@ -73,7 +74,7 @@ public void testInfiniteHttpCallGetSubscriptions() throws Exception { } @Test - public void testInfiniteHttpCallGetSubscriptions2() throws Exception { + public void testInfiniteHttpCallGetOrCreateSubscriptions2() throws Exception { final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0-abc"; Producer producer = pulsarClient.newProducer(Schema.STRING) @@ -82,13 +83,14 @@ public void testInfiniteHttpCallGetSubscriptions2() throws Exception { // Do test. admin.topics().getSubscriptions(topicName); + admin.topics().createSubscription(topicName, "s1", MessageId.earliest); // cleanup. producer.close(); } @Test - public void testInfiniteHttpCallGetSubscriptions3() throws Exception { + public void testInfiniteHttpCallGetOrCreateSubscriptions3() throws Exception { final String randomStr = UUID.randomUUID().toString().replaceAll("-", ""); final String topicName = "persistent://my-property/my-ns/tp1_" + randomStr + "-partition-0"; Producer producer = pulsarClient.newProducer(Schema.STRING) @@ -97,6 +99,7 @@ public void testInfiniteHttpCallGetSubscriptions3() throws Exception { // Do test. admin.topics().getSubscriptions(topicName); + admin.topics().createSubscription(topicName, "s1", MessageId.earliest); // cleanup. producer.close(); From 020b1dbcbdadd64fa74e27cfa97ab8e40ecc5909 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Fri, 2 Feb 2024 10:12:19 +0800 Subject: [PATCH 438/494] [improve][broker] Do not close the socket if lookup failed due to LockBusyException (#21993) --- .../pulsar/broker/lookup/TopicLookupBase.java | 8 +++- .../client/api/BrokerServiceLookupTest.java | 39 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index c4a39cd0d4455..7b2c777414884 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -30,6 +30,7 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationDataSource; @@ -333,13 +334,16 @@ public static CompletableFuture lookupTopicAsync(PulsarService pulsarSe private static void handleLookupError(CompletableFuture lookupFuture, String topicName, String clientAppId, long requestId, Throwable ex){ - final Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); + Throwable unwrapEx = FutureUtil.unwrapCompletionException(ex); final String errorMsg = unwrapEx.getMessage(); + if (unwrapEx instanceof PulsarServerException) { + unwrapEx = FutureUtil.unwrapCompletionException(unwrapEx.getCause()); + } if (unwrapEx instanceof IllegalStateException) { // Current broker still hold the bundle's lock, but the bundle is being unloading. log.info("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); lookupFuture.complete(newLookupErrorResponse(ServerError.MetadataError, errorMsg, requestId)); - } else if (unwrapEx instanceof MetadataStoreException){ + } else if (unwrapEx instanceof MetadataStoreException) { // Load bundle ownership or acquire lock failed. // Differ with "IllegalStateException", print warning log. log.warn("Failed to lookup {} for topic {} with error {}", clientAppId, topicName, errorMsg); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java index dab4fe9087e79..0a4c5b7a318b3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/BrokerServiceLookupTest.java @@ -71,6 +71,7 @@ import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit; +import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.namespace.OwnedBundle; import org.apache.pulsar.broker.namespace.OwnershipCache; @@ -1208,4 +1209,42 @@ private void makeAcquireBundleLockSuccess() throws Exception { mockZooKeeper.unsetAlwaysFail(); } } + + @Test(timeOut = 30000) + public void testLookupConnectionNotCloseIfFailedToAcquireOwnershipOfBundle() throws Exception { + String tpName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); + admin.topics().createNonPartitionedTopic(tpName); + final var pulsarClientImpl = (PulsarClientImpl) pulsarClient; + final var cache = pulsar.getNamespaceService().getOwnershipCache(); + final var bundle = pulsar.getNamespaceService().getBundle(TopicName.get(tpName)); + final var value = cache.getOwnerAsync(bundle).get().orElse(null); + assertNotNull(value); + + cache.invalidateLocalOwnerCache(); + final var lock = pulsar.getCoordinationService().getLockManager(NamespaceEphemeralData.class) + .acquireLock(ServiceUnitUtils.path(bundle), new NamespaceEphemeralData()).join(); + lock.updateValue(null); + log.info("Updated bundle {} with null", bundle.getBundleRange()); + + // wait for the system topic reader to __change_events is closed, otherwise the test will be affected + Thread.sleep(500); + + final var future = pulsarClientImpl.getLookup().getBroker(TopicName.get(tpName)); + final var cnx = pulsarClientImpl.getCnxPool().getConnections().stream().findAny() + .map(CompletableFuture::join).orElse(null); + assertNotNull(cnx); + + try { + future.get(); + fail(); + } catch (ExecutionException e) { + log.info("getBroker failed with {}: {}", e.getCause().getClass().getName(), e.getMessage()); + assertTrue(e.getCause() instanceof PulsarClientException.BrokerMetadataException); + assertTrue(cnx.ctx().channel().isActive()); + lock.updateValue(value); + lock.release(); + assertTrue(e.getMessage().contains("Failed to acquire ownership")); + pulsarClientImpl.getLookup().getBroker(TopicName.get(tpName)).get(); + } + } } From b796fc8f46ca1ab0ae55c1366c3e16a5b4e5c6ea Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Tue, 30 Jan 2024 23:40:09 +0800 Subject: [PATCH 439/494] [fix] [broker] add timeout for health check read. (#21990) --- .../pulsar/broker/admin/impl/BrokersBase.java | 13 +++- .../broker/admin/AdminApiHealthCheckTest.java | 63 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index 1234e9d49f4c6..57650758bbc70 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -26,6 +26,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,6 +35,7 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -81,6 +83,12 @@ public class BrokersBase extends AdminResource { // log a full thread dump when a deadlock is detected in healthcheck once every 10 minutes // to prevent excessive logging private static final long LOG_THREADDUMP_INTERVAL_WHEN_DEADLOCK_DETECTED = 600000L; + // there is a timeout of 60 seconds default in the client(readTimeoutMs), so we need to set the timeout + // a bit shorter than 60 seconds to avoid the client timeout exception thrown before the server timeout exception. + // or we can't propagate the server timeout exception to the client. + private static final Duration HEALTH_CHECK_READ_TIMEOUT = Duration.ofSeconds(58); + private static final TimeoutException HEALTH_CHECK_TIMEOUT_EXCEPTION = + FutureUtil.createTimeoutException("Timeout", BrokersBase.class, "healthCheckRecursiveReadNext(...)"); private volatile long threadDumpLoggedTimestamp; @GET @@ -435,7 +443,10 @@ private CompletableFuture internalRunHealthCheck(TopicVersion topicVersion }); throw FutureUtil.wrapToCompletionException(createException); }).thenCompose(reader -> producer.sendAsync(messageStr) - .thenCompose(__ -> healthCheckRecursiveReadNext(reader, messageStr)) + .thenCompose(__ -> FutureUtil.addTimeoutHandling( + healthCheckRecursiveReadNext(reader, messageStr), + HEALTH_CHECK_READ_TIMEOUT, pulsar().getBrokerService().executor(), + () -> HEALTH_CHECK_TIMEOUT_EXCEPTION)) .whenComplete((__, ex) -> { closeAndReCheck(producer, reader, topicOptional.get(), subscriptionName) .whenComplete((unused, innerEx) -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java index a780f889de85f..357422b11f6ce 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiHealthCheckTest.java @@ -23,6 +23,7 @@ import static org.testng.Assert.assertTrue; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; +import java.lang.reflect.Field; import java.time.Duration; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -31,13 +32,21 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ProducerBuilderImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.compaction.Compactor; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.springframework.util.CollectionUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -236,4 +245,58 @@ public void testHealthCheckupV2() throws Exception { )) ); } + + class DummyProducerBuilder extends ProducerBuilderImpl { + // This is a dummy producer builder to test the health check timeout + // the producer constructed by this builder will not send any message + public DummyProducerBuilder(PulsarClientImpl client, Schema schema) { + super(client, schema); + } + + @Override + public CompletableFuture> createAsync() { + CompletableFuture> future = new CompletableFuture<>(); + super.createAsync().thenAccept(producer -> { + Producer spyProducer = Mockito.spy(producer); + Mockito.doReturn(CompletableFuture.completedFuture(MessageId.earliest)) + .when(spyProducer).sendAsync(Mockito.any()); + future.complete(spyProducer); + }).exceptionally(ex -> { + future.completeExceptionally(ex); + return null; + }); + return future; + } + } + + @Test + public void testHealthCheckTimeOut() throws Exception { + final String testHealthCheckTopic = String.format("persistent://pulsar/localhost:%s/healthcheck", + pulsar.getConfig().getWebServicePort().get()); + PulsarClient client = pulsar.getClient(); + PulsarClient spyClient = Mockito.spy(client); + Mockito.doReturn(new DummyProducerBuilder<>((PulsarClientImpl) spyClient, Schema.BYTES)) + .when(spyClient).newProducer(Schema.STRING); + // use reflection to replace the client in the broker + Field field = PulsarService.class.getDeclaredField("client"); + field.setAccessible(true); + field.set(pulsar, spyClient); + try { + admin.brokers().healthcheck(TopicVersion.V2); + throw new Exception("Should not reach here"); + } catch (PulsarAdminException e) { + log.info("Exception caught", e); + assertTrue(e.getMessage().contains("LowOverheadTimeoutException")); + } + // To ensure we don't have any subscription, the producers and readers are closed. + Awaitility.await().untilAsserted(() -> + assertTrue(CollectionUtils.isEmpty(admin.topics() + .getSubscriptions(testHealthCheckTopic).stream() + // All system topics are using compaction, even though is not explicitly set in the policies. + .filter(v -> !v.equals(Compactor.COMPACTION_SUBSCRIPTION)) + .collect(Collectors.toList()) + )) + ); + } + } From 4ae07a9fb79bb57fef85f2bbc42f173b3f2ddf5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=A7=E6=98=93=E5=AE=A2?= Date: Tue, 30 Jan 2024 19:20:29 +0800 Subject: [PATCH 440/494] [fix][client] Fix ConsumerBuilderImpl#subscribe silent stuck when using pulsar-client:3.0.x with jackson-annotations prior to 2.12.0 (#21985) ### Motivation In summary, `jackson-annotations:2.12.0` or later is now required for `pulsar-client 3.0.x`, and this also applies to versions `3.1.x` and `3.2.x`. Otherwise, `ConsumerBuilderImpl#subscribe` may become stuck without displaying any error message. ### Modifications Modify the `whenComplete` to a combination of `thenAccept` and `exceptionally`. The modification is harmless. --- .../client/impl/MultiTopicsConsumerImpl.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index a0cbcc18e65f1..4c5d9f3ad7e94 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -996,13 +996,13 @@ CompletableFuture subscribeAsync(String topicName, int numberPartitions) { private void subscribeTopicPartitions(CompletableFuture subscribeResult, String topicName, int numPartitions, boolean createIfDoesNotExist) { - client.preProcessSchemaBeforeSubscribe(client, schema, topicName).whenComplete((schema, cause) -> { - if (null == cause) { - doSubscribeTopicPartitions(schema, subscribeResult, topicName, numPartitions, createIfDoesNotExist); - } else { - subscribeResult.completeExceptionally(cause); - } - }); + client.preProcessSchemaBeforeSubscribe(client, schema, topicName) + .thenAccept(schema -> { + doSubscribeTopicPartitions(schema, subscribeResult, topicName, numPartitions, createIfDoesNotExist); + }).exceptionally(cause -> { + subscribeResult.completeExceptionally(cause); + return null; + }); } private void doSubscribeTopicPartitions(Schema schema, From 5a2ad2842f264e0d0f1d9e290b36ff9aa7647b4f Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Mon, 29 Jan 2024 20:16:05 -0800 Subject: [PATCH 441/494] [fix][broker] Fix schema deletion error when deleting a partitioned topic with many partitions and schema (#21977) --- .../pulsar/broker/service/BrokerService.java | 29 +++++++++---------- .../schema/BookkeeperSchemaStorage.java | 6 ++-- .../tests/integration/schema/SchemaTest.java | 11 +++++++ 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index ceaec96b2a699..d9cea76ce9efe 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -121,8 +121,6 @@ import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.SystemTopic; import org.apache.pulsar.broker.service.plugin.EntryFilterProvider; -import org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage; -import org.apache.pulsar.broker.service.schema.SchemaRegistryService; import org.apache.pulsar.broker.stats.ClusterReplicationMetrics; import org.apache.pulsar.broker.stats.prometheus.metrics.ObserverGauge; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; @@ -3537,22 +3535,21 @@ public CompletableFuture deleteTopicPolicies(TopicName topicName) { } public CompletableFuture deleteSchema(TopicName topicName) { + // delete schema at the upper level when deleting the partitioned topic. + if (topicName.isPartitioned()) { + return CompletableFuture.completedFuture(null); + } String base = topicName.getPartitionedTopicName(); String id = TopicName.get(base).getSchemaName(); - SchemaRegistryService schemaRegistryService = getPulsar().getSchemaRegistryService(); - return BookkeeperSchemaStorage.ignoreUnrecoverableBKException(schemaRegistryService.getSchema(id)) - .thenCompose(schema -> { - if (schema != null) { - // It's different from `SchemasResource.deleteSchema` - // because when we delete a topic, the schema - // history is meaningless. But when we delete a schema of a topic, a new schema could be - // registered in the future. - log.info("Delete schema storage of id: {}", id); - return getPulsar().getSchemaRegistryService().deleteSchemaStorage(id); - } else { - return CompletableFuture.completedFuture(null); - } - }); + return getPulsar().getSchemaRegistryService().deleteSchemaStorage(id).whenComplete((vid, ex) -> { + if (vid != null && ex == null) { + // It's different from `SchemasResource.deleteSchema` + // because when we delete a topic, the schema + // history is meaningless. But when we delete a schema of a topic, a new schema could be + // registered in the future. + log.info("Deleted schema storage of id: {}", id); + } + }); } private CompletableFuture checkMaxTopicsPerNamespace(TopicName topicName, int numPartitions) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java index 78e30f6fff827..c509764bf6710 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/BookkeeperSchemaStorage.java @@ -707,7 +707,8 @@ public static Exception bkException(String operation, int rc, long ledgerId, lon message += " - entry=" + entryId; } boolean recoverable = rc != BKException.Code.NoSuchLedgerExistsException - && rc != BKException.Code.NoSuchEntryException; + && rc != BKException.Code.NoSuchEntryException + && rc != BKException.Code.NoSuchLedgerExistsOnMetadataServerException; return new SchemaException(recoverable, message); } @@ -716,7 +717,8 @@ public static CompletableFuture ignoreUnrecoverableBKException(Completabl if (t.getCause() != null && (t.getCause() instanceof SchemaException) && !((SchemaException) t.getCause()).isRecoverable()) { - // Meeting NoSuchLedgerExistsException or NoSuchEntryException when reading schemas in + // Meeting NoSuchLedgerExistsException, NoSuchEntryException or + // NoSuchLedgerExistsOnMetadataServerException when reading schemas in // bookkeeper. This also means that the data has already been deleted by other operations // in deleting schema. if (log.isDebugEnabled()) { diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java index 8bb6de74c661d..d0421063b2d90 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java @@ -31,6 +31,8 @@ import org.apache.pulsar.client.api.schema.SchemaDefinition; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; import org.apache.pulsar.tests.integration.schema.Schemas.Person; import org.apache.pulsar.tests.integration.schema.Schemas.PersonConsumeSchema; import org.apache.pulsar.tests.integration.schema.Schemas.Student; @@ -316,5 +318,14 @@ public void testPrimitiveSchemaTypeCompatibilityCheck() { } + @Test + public void testDeletePartitionedTopicWhenTopicReferenceIsNotReady() throws Exception { + final String topic = "persistent://public/default/tp-ref"; + admin.topics().createPartitionedTopic(topic, 20); + admin.schemas().createSchema(topic, + SchemaInfo.builder().type(SchemaType.STRING).schema(new byte[0]).build()); + admin.topics().deletePartitionedTopic(topic, false); + } + } From 90769519a761576bc96a5e2ff20802cffd973cb4 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Tue, 30 Jan 2024 19:34:01 +0800 Subject: [PATCH 442/494] [improve] [proxy] Add a check for brokerServiceURL that does not support multi uri yet (#21972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation At the beginning of the design, these two configurations(`brokerServiceURL & brokerServiceURLTLS`) do not support setting multiple broker addresses, which should instead be set to a “discovery service provider.” see: https://github.com/apache/pulsar/pull/1002 and https://github.com/apache/pulsar/pull/14682 Users will get the below error if they set A to a multi-broker URLs ``` "2024-01-09 00:20:10,261 -0800 [pulsar-proxy-io-4-7] WARN io.netty.channel.DefaultChannelPipeline - An exceptionCaught() event was fired, and it reached at the tail of the pipeline. It usually means the last handler in the pipeline did not handle the exception. java.lang.IllegalArgumentException: port out of range:-1 at java.net.InetSocketAddress.checkPort(InetSocketAddress.java:143) ~[?:?] at java.net.InetSocketAddress.createUnresolved(InetSocketAddress.java:254) ~[?:?] at org.apache.pulsar.proxy.server.LookupProxyHandler.getAddr(LookupProxyHandler.java:432) ~[org.apache.pulsar-pulsar-proxy-2.9.0.jar:2.9.0] at org.apache.pulsar.proxy.server.LookupProxyHandler.handleGetSchema(LookupProxyHandler.java:357) ~[org.apache.pulsar-pulsar-proxy-2.9.0.jar:2.9.0] at org.apache.pulsar.proxy.server.ProxyConnection.handleGetSchema(ProxyConnection.java:463) ~[org.apache.pulsar-pulsar-proxy-2.9.0.jar:2.9.0] at org.apache.pulsar.common.protocol.PulsarDecoder.channelRead(PulsarDecoder.java:326) ~[io.streamnative-pulsar-common-2.9.2.12.jar:2.9.2.12] at org.apache.pulsar.proxy.server.ProxyConnection.channelRead(ProxyConnection.java:221) ~[org.apache.pulsar-pulsar-proxy-2.9.0.jar:2.9.0] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327) ~[io.netty-netty-codec-4.1.74.Final.jar:4.1.74.Final] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:299) ~[io.netty-netty-codec-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1372) ~[io.netty-netty-handler-4.1.74.Final.jar:4.1.74.Final] at io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1246) ~[io.netty-netty-handler-4.1.74.Final.jar:4.1.74.Final] at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1286) ~[io.netty-netty-handler-4.1.74.Final.jar:4.1.74.Final] at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[io.netty-netty-codec-4.1.74.Final.jar:4.1.74.Final] at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[io.netty-netty-codec-4.1.74.Final.jar:4.1.74.Final] at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[io.netty-netty-codec-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[io.netty-netty-transport-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:795) ~[io.netty-netty-transport-classes-epoll-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:480) ~[io.netty-netty-transport-classes-epoll-4.1.74.Final.jar:4.1.74.Final] at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:378) ~[io.netty-netty-transport-classes-epoll-4.1.74.Final.jar:4.1.74.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) ~[io.netty-netty-common-4.1.74.Final.jar:4.1.74.Final] at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[io.netty-netty-common-4.1.74.Final.jar:4.1.74.Final] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[io.netty-netty-common-4.1.74.Final.jar:4.1.74.Final] ``` ### Modifications - Improve the description - Add a check to prevent wrong settings --- conf/proxy.conf | 10 +- .../proxy/server/ProxyConfiguration.java | 20 ++- .../proxy/server/ProxyServiceStarter.java | 24 +++- .../proxy/server/ProxyConfigurationTest.java | 119 ++++++++++++++++++ .../proxy/server/ProxyServiceStarterTest.java | 2 +- 5 files changed, 163 insertions(+), 12 deletions(-) diff --git a/conf/proxy.conf b/conf/proxy.conf index 5d9bd584ff766..613022b5dc2e1 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -28,17 +28,19 @@ metadataStoreUrl= # The metadata store URL for the configuration data. If empty, we fall back to use metadataStoreUrl configurationMetadataStoreUrl= -# If Service Discovery is Disabled this url should point to the discovery service provider. +# If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the discovery service +# provider, and does not support multi urls yet. # The URL must begin with pulsar:// for plaintext or with pulsar+ssl:// for TLS. brokerServiceURL= brokerServiceURLTLS= -# These settings are unnecessary if `zookeeperServers` is specified +# If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the discovery service +# provider, and does not support multi urls yet. brokerWebServiceURL= brokerWebServiceURLTLS= -# If function workers are setup in a separate cluster, configure the following 2 settings -# to point to the function workers cluster +# If function workers are setup in a separate cluster, configure the following 2 settings. This url should point to +# the discovery service provider of the function workers cluster, and does not support multi urls yet. functionWorkerWebServiceURL= functionWorkerWebServiceURLTLS= diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 7178a0ceda4db..db2969e3c3920 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -173,35 +173,43 @@ public class ProxyConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The service url points to the broker cluster. URL must have the pulsar:// prefix." + doc = "If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the" + + " discovery service provider." + + " URL must have the pulsar:// prefix. And does not support multi url yet." ) private String brokerServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls service url points to the broker cluster. URL must have the pulsar+ssl:// prefix." + doc = "If does not set metadataStoreUrl or configurationMetadataStoreUrl, this url should point to the" + + " discovery service provider." + + " URL must have the pulsar+ssl:// prefix. And does not support multi url yet." ) private String brokerServiceURLTLS; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The web service url points to the broker cluster" + doc = "The web service url points to the discovery service provider of the broker cluster, and does not support" + + " multi url yet." ) private String brokerWebServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls web service url points to the broker cluster" + doc = "The tls web service url points to the discovery service provider of the broker cluster, and does not" + + " support multi url yet." ) private String brokerWebServiceURLTLS; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The web service url points to the function worker cluster." + doc = "The web service url points to the discovery service provider of the function worker cluster, and does" + + " not support multi url yet." + " Only configure it when you setup function workers in a separate cluster" ) private String functionWorkerWebServiceURL; @FieldContext( category = CATEGORY_BROKER_DISCOVERY, - doc = "The tls web service url points to the function worker cluster." + doc = "The tls web service url points to the discovery service provider of the function worker cluster, and" + + " does not support multi url yet." + " Only configure it when you setup function workers in a separate cluster" ) private String functionWorkerWebServiceURLTLS; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index 84f83a901a3e4..0cea5e9d18439 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -161,11 +161,28 @@ public ProxyServiceStarter(String[] args) throws Exception { if (isNotBlank(config.getBrokerServiceURL())) { checkArgument(config.getBrokerServiceURL().startsWith("pulsar://"), "brokerServiceURL must start with pulsar://"); + ensureUrlNotContainsComma("brokerServiceURL", config.getBrokerServiceURL()); } - if (isNotBlank(config.getBrokerServiceURLTLS())) { checkArgument(config.getBrokerServiceURLTLS().startsWith("pulsar+ssl://"), "brokerServiceURLTLS must start with pulsar+ssl://"); + ensureUrlNotContainsComma("brokerServiceURLTLS", config.getBrokerServiceURLTLS()); + } + + if (isNotBlank(config.getBrokerWebServiceURL())) { + ensureUrlNotContainsComma("brokerWebServiceURL", config.getBrokerWebServiceURL()); + } + if (isNotBlank(config.getBrokerWebServiceURLTLS())) { + ensureUrlNotContainsComma("brokerWebServiceURLTLS", config.getBrokerWebServiceURLTLS()); + } + + if (isNotBlank(config.getFunctionWorkerWebServiceURL())) { + ensureUrlNotContainsComma("functionWorkerWebServiceURLTLS", + config.getFunctionWorkerWebServiceURL()); + } + if (isNotBlank(config.getFunctionWorkerWebServiceURLTLS())) { + ensureUrlNotContainsComma("functionWorkerWebServiceURLTLS", + config.getFunctionWorkerWebServiceURLTLS()); } if ((isBlank(config.getBrokerServiceURL()) && isBlank(config.getBrokerServiceURLTLS())) @@ -186,6 +203,11 @@ public ProxyServiceStarter(String[] args) throws Exception { } } + private void ensureUrlNotContainsComma(String paramName, String paramValue) { + checkArgument(!paramValue.contains(","), paramName + " does not support multi urls yet," + + " it should point to the discovery service provider."); + } + public static void main(String[] args) throws Exception { ProxyServiceStarter serviceStarter = new ProxyServiceStarter(args); try { diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java index 97a73c20b60d0..a9a562e04c899 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyConfigurationTest.java @@ -20,6 +20,8 @@ import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.testng.annotations.Test; import java.beans.Introspector; @@ -36,6 +38,8 @@ import java.util.Properties; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; @Test(groups = "broker") public class ProxyConfigurationTest { @@ -134,4 +138,119 @@ public void testConvert() throws IOException { } } + @Test + public void testBrokerUrlCheck() throws IOException { + ProxyConfiguration configuration = new ProxyConfiguration(); + // brokerServiceURL must start with pulsar:// + configuration.setBrokerServiceURL("127.0.0.1:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURL must start with pulsar://"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("brokerServiceURL must start with pulsar://")); + } + } + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650"); + + // brokerServiceURLTLS must start with pulsar+ssl:// + configuration.setBrokerServiceURLTLS("pulsar://127.0.0.1:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURLTLS must start with pulsar+ssl://"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("brokerServiceURLTLS must start with pulsar+ssl://")); + } + } + + // brokerServiceURL did not support multi urls yet. + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650,pulsar://127.0.0.2:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerServiceURL("pulsar://127.0.0.1:6650"); + + // brokerServiceURLTLS did not support multi urls yet. + configuration.setBrokerServiceURLTLS("pulsar+ssl://127.0.0.1:6650,pulsar+ssl:127.0.0.2:6650"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerServiceURLTLS("pulsar+ssl://127.0.0.1:6650"); + + // brokerWebServiceURL did not support multi urls yet. + configuration.setBrokerWebServiceURL("http://127.0.0.1:8080,http://127.0.0.2:8080"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerWebServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerWebServiceURL("http://127.0.0.1:8080"); + + // brokerWebServiceURLTLS did not support multi urls yet. + configuration.setBrokerWebServiceURLTLS("https://127.0.0.1:443,https://127.0.0.2:443"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("brokerWebServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setBrokerWebServiceURLTLS("https://127.0.0.1:443"); + + // functionWorkerWebServiceURL did not support multi urls yet. + configuration.setFunctionWorkerWebServiceURL("http://127.0.0.1:8080,http://127.0.0.2:8080"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("functionWorkerWebServiceURL does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setFunctionWorkerWebServiceURL("http://127.0.0.1:8080"); + + // functionWorkerWebServiceURLTLS did not support multi urls yet. + configuration.setFunctionWorkerWebServiceURLTLS("http://127.0.0.1:443,http://127.0.0.2:443"); + try (MockedStatic theMock = Mockito.mockStatic(PulsarConfigurationLoader.class)) { + theMock.when(PulsarConfigurationLoader.create(Mockito.anyString(), Mockito.any())) + .thenReturn(configuration); + try { + new ProxyServiceStarter(ProxyServiceStarterTest.ARGS); + fail("functionWorkerWebServiceURLTLS does not support multi urls yet"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("does not support multi urls yet")); + } + } + configuration.setFunctionWorkerWebServiceURLTLS("http://127.0.0.1:443"); + } + } \ No newline at end of file diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java index def58be6df372..a9bead706a373 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceStarterTest.java @@ -45,7 +45,7 @@ public class ProxyServiceStarterTest extends MockedPulsarServiceBaseTest { - static final String[] ARGS = new String[]{"-c", "./src/test/resources/proxy.conf"}; + public static final String[] ARGS = new String[]{"-c", "./src/test/resources/proxy.conf"}; protected ProxyServiceStarter serviceStarter; protected String serviceUrl; From 9baebfa65f3514981810db70af5ec130bb413643 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 31 Jan 2024 22:18:47 +0800 Subject: [PATCH 443/494] [fix][fn] Use unified PackageManagement service to download packages (#21955) --- .../apache/pulsar/functions/worker/rest/api/ComponentImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 585b54d846169..6bde2b4fc8819 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -1822,6 +1822,8 @@ protected File getPackageFile(String functionPkgUrl, String existingPackagePath, + "when getting %s package from %s", e.getMessage(), ComponentTypeUtils.toString(componentType), functionPkgUrl)); } + } else if (Utils.hasPackageTypePrefix(existingPackagePath)) { + componentPackageFile = getPackageFile(existingPackagePath); } else if (uploadedInputStream != null) { componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream); } else if (!existingPackagePath.startsWith(Utils.BUILTIN)) { From a209984a2afcc84379e6b784a98629acb3af078b Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 25 Jan 2024 17:02:26 +0800 Subject: [PATCH 444/494] [fix] [broker] Replication stopped due to unload topic failed (#21947) ### Motivation **Steps to reproduce the issue** - Enable replication. - Send `10` messages to the local cluster then close the producer. - Call `pulsar-admin topics unload ` and get an error due to the internal producer of the replicator close failing. - The topic closed failed, so we assumed the topic could work as expected, but the replication stopped. **Root cause** - `pulsar-admin topics unload ` will wait for the clients(including `consumers & producers & replicators`) to close successfully, and it will fail if clients can not be closed successfully. - `replicator.producer` close failed causing the Admin API to fail, but there is a scheduled task that will retry to close `replicator.producer` which causes replication to stop. see https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java#L209 ### Modifications since the "replicator.producer.closeAsync()" will retry after it fails, the topic unload should be successful. --- .../broker/service/AbstractReplicator.java | 3 +- .../broker/service/OneWayReplicatorTest.java | 70 ++++++++++++++++--- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java index 539f178d665cc..6d0dc2dc9e158 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractReplicator.java @@ -193,7 +193,7 @@ protected synchronized CompletableFuture closeProducerAsync() { return CompletableFuture.completedFuture(null); } CompletableFuture future = producer.closeAsync(); - future.thenRun(() -> { + return future.thenRun(() -> { STATE_UPDATER.set(this, State.Stopped); this.producer = null; // deactivate further read @@ -208,7 +208,6 @@ protected synchronized CompletableFuture closeProducerAsync() { brokerService.executor().schedule(this::closeProducerAsync, waitTimeMs, TimeUnit.MILLISECONDS); return null; }); - return future; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index c8948f91bc38d..61105ebbac5d2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -21,17 +21,23 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; +import java.lang.reflect.Field; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.persistent.PersistentReplicator; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.client.impl.ProducerImpl; import org.apache.pulsar.common.policies.data.TopicStats; import org.junit.Assert; import org.awaitility.Awaitility; +import org.mockito.Mockito; +import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -51,6 +57,29 @@ public void cleanup() throws Exception { super.cleanup(); } + private void waitReplicatorStarted(String topicName) { + Awaitility.await().untilAsserted(() -> { + Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get(); + assertTrue(topicOptional2.isPresent()); + PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); + assertFalse(persistentTopic2.getProducers().isEmpty()); + }); + } + + /** + * Override "AbstractReplicator.producer" by {@param producer} and return the original value. + */ + private ProducerImpl overrideProducerForReplicator(AbstractReplicator replicator, ProducerImpl newProducer) + throws Exception { + Field producerField = AbstractReplicator.class.getDeclaredField("producer"); + producerField.setAccessible(true); + ProducerImpl originalValue = (ProducerImpl) producerField.get(replicator); + synchronized (replicator) { + producerField.set(replicator, newProducer); + } + return originalValue; + } + @Test public void testReplicatorProducerStatInTopic() throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); @@ -86,18 +115,13 @@ public void testReplicatorProducerStatInTopic() throws Exception { public void testCreateRemoteConsumerFirst() throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); - // Wait for replicator started. - Awaitility.await().untilAsserted(() -> { - Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get(); - assertTrue(topicOptional2.isPresent()); - PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); - assertFalse(persistentTopic2.getProducers().isEmpty()); - }); + // The topic in cluster2 has a replicator created producer(schema Auto_Produce), but does not have any schema。 // Verify: the consumer of this cluster2 can create successfully. Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topicName).subscriptionName("s1") .subscribe();; - + // Wait for replicator started. + waitReplicatorStarted(topicName); // cleanup. producer1.close(); consumer2.close(); @@ -106,4 +130,34 @@ public void testCreateRemoteConsumerFirst() throws Exception { admin2.topics().delete(topicName); }); } + + @Test + public void testTopicCloseWhenInternalProducerCloseErrorOnce() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + admin1.topics().createNonPartitionedTopic(topicName); + // Wait for replicator started. + waitReplicatorStarted(topicName); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + PersistentReplicator replicator = + (PersistentReplicator) persistentTopic.getReplicators().values().iterator().next(); + // Mock an error when calling "replicator.disconnect()" + ProducerImpl mockProducer = Mockito.mock(ProducerImpl.class); + Mockito.when(mockProducer.closeAsync()).thenReturn(CompletableFuture.failedFuture(new Exception("mocked ex"))); + ProducerImpl originalProducer = overrideProducerForReplicator(replicator, mockProducer); + // Verify: since the "replicator.producer.closeAsync()" will retry after it failed, the topic unload should be + // successful. + admin1.topics().unload(topicName); + // Verify: After "replicator.producer.closeAsync()" retry again, the "replicator.producer" will be closed + // successful. + overrideProducerForReplicator(replicator, originalProducer); + Awaitility.await().untilAsserted(() -> { + Assert.assertFalse(replicator.isConnected()); + }); + // cleanup. + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } } From e8bb6ca492c2a82dc3bec4e24453a190ccfdf2e5 Mon Sep 17 00:00:00 2001 From: Zixuan Liu Date: Fri, 19 Jan 2024 11:18:02 +0800 Subject: [PATCH 445/494] [fix][broker] Fix getMessageById throws 500 (#21919) Signed-off-by: Zixuan Liu --- .../admin/impl/PersistentTopicsBase.java | 3 +++ .../broker/admin/PersistentTopicsTest.java | 21 ++++++------------- .../client/admin/internal/TopicsImpl.java | 16 +------------- 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 53c412bcf1d6b..780537b9cb06b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -2915,6 +2915,9 @@ protected CompletableFuture internalGetMessageById(long ledgerId, long @Override public void readEntryFailed(ManagedLedgerException exception, Object ctx) { + if (exception instanceof ManagedLedgerException.LedgerNotExistException) { + throw new RestException(Status.NOT_FOUND, "Message id not found"); + } throw new RestException(exception); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java index 8dee90af4afb4..80a493188d112 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/PersistentTopicsTest.java @@ -1370,21 +1370,12 @@ public void testGetMessageById() throws Exception { Message message2 = admin.topics().getMessageById(topicName2, id2.getLedgerId(), id2.getEntryId()); Assert.assertEquals(message2.getData(), data2.getBytes()); - Message message3 = null; - try { - message3 = admin.topics().getMessageById(topicName2, id1.getLedgerId(), id1.getEntryId()); - Assert.fail(); - } catch (Exception e) { - Assert.assertNull(message3); - } - - Message message4 = null; - try { - message4 = admin.topics().getMessageById(topicName1, id2.getLedgerId(), id2.getEntryId()); - Assert.fail(); - } catch (Exception e) { - Assert.assertNull(message4); - } + Assert.expectThrows(PulsarAdminException.NotFoundException.class, () -> { + admin.topics().getMessageById(topicName2, id1.getLedgerId(), id1.getEntryId()); + }); + Assert.expectThrows(PulsarAdminException.NotFoundException.class, () -> { + admin.topics().getMessageById(topicName1, id2.getLedgerId(), id2.getEntryId()); + }); } @Test diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java index 33d1cd1785827..2d9754a0748df 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TopicsImpl.java @@ -964,21 +964,7 @@ public CompletableFuture truncateAsync(String topic) { @Override public CompletableFuture> getMessageByIdAsync(String topic, long ledgerId, long entryId) { - CompletableFuture> future = new CompletableFuture<>(); - getRemoteMessageById(topic, ledgerId, entryId).handle((r, ex) -> { - if (ex != null) { - if (ex instanceof NotFoundException) { - log.warn("Exception '{}' occurred while trying to get message.", ex.getMessage()); - future.complete(r); - } else { - future.completeExceptionally(ex); - } - return null; - } - future.complete(r); - return null; - }); - return future; + return getRemoteMessageById(topic, ledgerId, entryId); } private CompletableFuture> getRemoteMessageById(String topic, long ledgerId, long entryId) { From ac1f4992fecaf2d1a7d479a1f98e73f577048323 Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Wed, 31 Jan 2024 00:31:15 +0800 Subject: [PATCH 446/494] [fix][client] Fix multi-topics consumer could receive old messages after seek (#21945) --- .../client/impl/TopicsConsumerImplTest.java | 80 ++++++++++++++++++- .../client/impl/MultiTopicsConsumerImpl.java | 66 ++++++++++----- 2 files changed, 125 insertions(+), 21 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java index 51b32c2b44ecf..c343ab0d6e294 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java @@ -34,6 +34,7 @@ import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageIdAdv; +import org.apache.pulsar.client.api.MessageListener; import org.apache.pulsar.client.api.MessageRouter; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; @@ -57,22 +58,27 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; - import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.Set; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeSet; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -1394,4 +1400,76 @@ public Map getActiveConsumers() { } } + @DataProvider + public static Object[][] seekByFunction() { + return new Object[][] { + { true }, { false } + }; + } + + @Test(timeOut = 30000, dataProvider = "seekByFunction") + public void testSeekToNewerPosition(boolean seekByFunction) throws Exception { + final var topic1 = TopicName.get(newTopicName()).toString() + .replace("my-property", "public").replace("my-ns", "default"); + final var topic2 = TopicName.get(newTopicName()).toString() + .replace("my-property", "public").replace("my-ns", "default"); + @Cleanup final var producer1 = pulsarClient.newProducer(Schema.STRING).topic(topic1).create(); + @Cleanup final var producer2 = pulsarClient.newProducer(Schema.STRING).topic(topic2).create(); + producer1.send("1-0"); + producer2.send("2-0"); + producer1.send("1-1"); + producer2.send("2-1"); + final var consumer1 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + final var timestamps = new ArrayList(); + for (int i = 0; i < 4; i++) { + timestamps.add(consumer1.receive().getPublishTime()); + } + timestamps.sort(Comparator.naturalOrder()); + final var timestamp = timestamps.get(2); + consumer1.close(); + + final Function, CompletableFuture> seekAsync = consumer -> { + final var future = seekByFunction ? consumer.seekAsync(__ -> timestamp) : consumer.seekAsync(timestamp); + assertEquals(((ConsumerBase) consumer).getIncomingMessageSize(), 0L); + assertEquals(((ConsumerBase) consumer).getTotalIncomingMessages(), 0); + assertTrue(((ConsumerBase) consumer).getUnAckedMessageTracker().isEmpty()); + return future; + }; + + @Cleanup final var consumer2 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-2") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer2).get(); + final var values = new TreeSet(); + for (int i = 0; i < 2; i++) { + values.add(consumer2.receive().getValue()); + } + assertEquals(values, new TreeSet<>(Arrays.asList("1-1", "2-1"))); + + final var valuesInListener = new CopyOnWriteArrayList(); + @Cleanup final var consumer3 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-3") + .messageListener((MessageListener) (__, msg) -> valuesInListener.add(msg.getValue())) + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer3).get(); + if (valuesInListener.isEmpty()) { + Awaitility.await().untilAsserted(() -> assertEquals(valuesInListener.size(), 2)); + assertEquals(valuesInListener.stream().sorted().toList(), Arrays.asList("1-1", "2-1")); + } // else: consumer3 has passed messages to the listener before seek, in this case we cannot assume anything + + @Cleanup final var consumer4 = pulsarClient.newConsumer(Schema.STRING) + .topics(Arrays.asList(topic1, topic2)).subscriptionName("sub-4") + .ackTimeout(1, TimeUnit.SECONDS) + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe(); + seekAsync.apply(consumer4).get(); + final var valuesInReceiveAsync = new ArrayList(); + valuesInReceiveAsync.add(consumer4.receiveAsync().get().getValue()); + valuesInReceiveAsync.add(consumer4.receiveAsync().get().getValue()); + assertEquals(valuesInReceiveAsync.stream().sorted().toList(), Arrays.asList("1-1", "2-1")); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 4c5d9f3ad7e94..5e16dcb0f750a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -49,6 +49,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import javax.annotation.Nullable; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.client.api.BatchReceivePolicy; import org.apache.pulsar.client.api.Consumer; @@ -101,7 +102,8 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { private final MultiTopicConsumerStatsRecorderImpl stats; private final ConsumerConfigurationData internalConfig; - private volatile MessageIdAdv startMessageId; + private final MessageIdAdv startMessageId; + private volatile boolean duringSeek = false; private final long startMessageRollbackDurationInSec; MultiTopicsConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf, ExecutorProvider executorProvider, CompletableFuture> subscribeFuture, Schema schema, @@ -235,6 +237,10 @@ private void startReceivingMessages(List> newConsumers) { } private void receiveMessageFromConsumer(ConsumerImpl consumer, boolean batchReceive) { + if (duringSeek) { + log.info("[{}] Pause receiving messages for topic {} due to seek", subscription, consumer.getTopic()); + return; + } CompletableFuture>> messagesFuture; if (batchReceive) { messagesFuture = consumer.batchReceiveAsync().thenApply(msgs -> ((MessagesImpl) msgs).getMessageList()); @@ -252,8 +258,12 @@ private void receiveMessageFromConsumer(ConsumerImpl consumer, boolean batchR } // Process the message, add to the queue and trigger listener or async callback messages.forEach(msg -> { - if (isValidConsumerEpoch((MessageImpl) msg)) { + final boolean skipDueToSeek = duringSeek; + if (isValidConsumerEpoch((MessageImpl) msg) && !skipDueToSeek) { messageReceived(consumer, msg); + } else if (skipDueToSeek) { + log.info("[{}] [{}] Skip processing message {} received during seek", topic, subscription, + msg.getMessageId()); } }); @@ -746,17 +756,12 @@ public void seek(Function function) throws PulsarClientException @Override public CompletableFuture seekAsync(Function function) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumer -> futures.add(consumer.seekAsync(function))); - unAckedMessageTracker.clear(); - incomingMessages.clear(); - resetIncomingMessageSize(); - return FutureUtil.waitForAll(futures); + return seekAllAsync(consumer -> consumer.seekAsync(function)); } @Override public CompletableFuture seekAsync(MessageId messageId) { - final Consumer internalConsumer; + final ConsumerImpl internalConsumer; if (messageId instanceof TopicMessageId) { TopicMessageId topicMessageId = (TopicMessageId) messageId; internalConsumer = consumers.get(topicMessageId.getOwnerTopic()); @@ -773,25 +778,46 @@ public CompletableFuture seekAsync(MessageId messageId) { ); } - final CompletableFuture seekFuture; if (internalConsumer == null) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumerImpl -> futures.add(consumerImpl.seekAsync(messageId))); - seekFuture = FutureUtil.waitForAll(futures); + return seekAllAsync(consumer -> consumer.seekAsync(messageId)); } else { - seekFuture = internalConsumer.seekAsync(messageId); + return seekAsyncInternal(Collections.singleton(internalConsumer), __ -> __.seekAsync(messageId)); } + } + + @Override + public CompletableFuture seekAsync(long timestamp) { + return seekAllAsync(consumer -> consumer.seekAsync(timestamp)); + } + private CompletableFuture seekAsyncInternal(Collection> consumers, + Function, CompletableFuture> seekFunc) { + beforeSeek(); + final CompletableFuture future = new CompletableFuture<>(); + FutureUtil.waitForAll(consumers.stream().map(seekFunc).collect(Collectors.toList())) + .whenComplete((__, e) -> afterSeek(future, e)); + return future; + } + + private CompletableFuture seekAllAsync(Function, CompletableFuture> seekFunc) { + return seekAsyncInternal(consumers.values(), seekFunc); + } + + private void beforeSeek() { + duringSeek = true; unAckedMessageTracker.clear(); clearIncomingMessages(); - return seekFuture; } - @Override - public CompletableFuture seekAsync(long timestamp) { - List> futures = new ArrayList<>(consumers.size()); - consumers.values().forEach(consumer -> futures.add(consumer.seekAsync(timestamp))); - return FutureUtil.waitForAll(futures); + private void afterSeek(CompletableFuture seekFuture, @Nullable Throwable throwable) { + duringSeek = false; + log.info("[{}] Resume receiving messages for {} since seek is done", subscription, consumers.keySet()); + startReceivingMessages(new ArrayList<>(consumers.values())); + if (throwable == null) { + seekFuture.complete(null); + } else { + seekFuture.completeExceptionally(throwable); + } } @Override From 55512c81a1d5eaa15a75edaa3a4702b16d3c44bb Mon Sep 17 00:00:00 2001 From: hrzzzz <64506104+hrzzzz@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:23:53 +0800 Subject: [PATCH 447/494] [fix][broker] Fix deadlock while skip non-recoverable ledgers. (#21915) ### Motivation The sequence of events leading to the deadlock when methods from org.apache.bookkeeper.mledger.impl.ManagedCursorImpl are invoked concurrently is as follows: 1. Thread A calls asyncDelete, which then goes on to internally call internalAsyncMarkDelete. This results in acquiring a lock on pendingMarkDeleteOps through synchronized (pendingMarkDeleteOps). 2. Inside internalAsyncMarkDelete, internalMarkDelete is called which subsequently calls persistPositionToLedger. At the start of persistPositionToLedger, buildIndividualDeletedMessageRanges is invoked, where it tries to acquire a read lock using lock.readLock().lock(). At this point, if the write lock is being held by another thread, Thread A will block waiting for the read lock. 3. Concurrently, Thread B executes skipNonRecoverableLedger which first obtains a write lock using lock.writeLock().lock() and then proceeds to call asyncDelete. 4. At this moment, Thread B already holds the write lock and is attempting to acquire the synchronized lock on pendingMarkDeleteOps that Thread A already holds, while Thread A is waiting for the read lock that Thread B needs to release. In code, the deadlock appears as follows: Thread A: synchronized (pendingMarkDeleteOps) -> lock.readLock().lock() (waiting) Thread B: lock.writeLock().lock() -> synchronized (pendingMarkDeleteOps) (waiting) ### Modifications Avoid using a long-range lock. Co-authored-by: ruihongzhou Co-authored-by: Jiwe Guo Co-authored-by: Lari Hotari --- .../mledger/impl/ManagedCursorImpl.java | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index d2d58e74b1ebb..fcf1a2a20104f 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -59,6 +59,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.LongStream; import org.apache.bookkeeper.client.AsyncCallback.CloseCallback; import org.apache.bookkeeper.client.AsyncCallback.OpenCallback; import org.apache.bookkeeper.client.BKException; @@ -2770,30 +2771,23 @@ public void skipNonRecoverableLedger(final long ledgerId){ if (ledgerInfo == null) { return; } - lock.writeLock().lock(); log.warn("[{}] [{}] Since the ledger [{}] is lost and the autoSkipNonRecoverableData is true, this ledger will" + " be auto acknowledge in subscription", ledger.getName(), name, ledgerId); - try { - for (int i = 0; i < ledgerInfo.getEntries(); i++) { - if (!individualDeletedMessages.contains(ledgerId, i)) { - asyncDelete(PositionImpl.get(ledgerId, i), new AsyncCallbacks.DeleteCallback() { - @Override - public void deleteComplete(Object ctx) { - // ignore. - } + asyncDelete(() -> LongStream.range(0, ledgerInfo.getEntries()) + .mapToObj(i -> (Position) PositionImpl.get(ledgerId, i)).iterator(), + new AsyncCallbacks.DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + // ignore. + } - @Override - public void deleteFailed(ManagedLedgerException ex, Object ctx) { - // The method internalMarkDelete already handled the failure operation. We only need to - // make sure the memory state is updated. - // If the broker crashed, the non-recoverable ledger will be detected again. - } - }, null); - } - } - } finally { - lock.writeLock().unlock(); - } + @Override + public void deleteFailed(ManagedLedgerException ex, Object ctx) { + // The method internalMarkDelete already handled the failure operation. We only need to + // make sure the memory state is updated. + // If the broker crashed, the non-recoverable ledger will be detected again. + } + }, null); } // ////////////////////////////////////////////////// From 20656534e0d26e169acffe066bfcdbf3392dba5f Mon Sep 17 00:00:00 2001 From: Yunze Xu Date: Tue, 16 Jan 2024 18:18:31 +0800 Subject: [PATCH 448/494] [improve][ci] Upgrade pulsar-client-python to 3.4.0 to avoid CVE-2023-1428 (#21899) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9250b11f4ecf2..08adf869b9875 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ flexible messaging model and an intuitive client API. ${maven.compiler.target} 8 - 3.2.0 + 3.4.0 **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java From a2ed6736c62078b6d03727cd5fd49ebc65a6ffd8 Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Sun, 21 Jan 2024 19:47:40 +0800 Subject: [PATCH 449/494] [fix] [broker] Fix write all compacted out entry into compacted topic (#21917) --- .../pulsar/client/impl/RawBatchConverter.java | 6 +- .../pulsar/compaction/CompactorTest.java | 56 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java index 1b1b2e3ebcdba..875c30f0f5e13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java @@ -130,7 +130,11 @@ public static Optional rebatchMessage(RawMessage msg, msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i); - if (!singleMessageMetadata.hasPartitionKey()) { + if (singleMessageMetadata.isCompactedOut()) { + // we may read compacted out message from the compacted topic + Commands.serializeSingleMessageInBatchWithPayload(emptyMetadata, + Unpooled.EMPTY_BUFFER, batchBuffer); + } else if (!singleMessageMetadata.hasPartitionKey()) { if (retainNullKey) { messagesRetained++; Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadata, diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java index 464f05186fa30..eab5f10bd295f 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java @@ -36,6 +36,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; @@ -46,10 +48,15 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.ConnectionPool; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawMessageImpl; @@ -177,6 +184,55 @@ public void testCompaction() throws Exception { compactAndVerify(topic, expected, true); } + @Test + public void testAllCompactedOut() throws Exception { + String topicName = "persistent://my-property/use/my-ns/testAllCompactedOut"; + // set retain null key to true + boolean oldRetainNullKey = pulsar.getConfig().isTopicCompactionRetainNullKey(); + pulsar.getConfig().setTopicCompactionRetainNullKey(true); + this.restartBroker(); + + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(true).topic(topicName).batchingMaxMessages(3).create(); + + producer.newMessage().key("K1").value("V1").sendAsync(); + producer.newMessage().key("K2").value("V2").sendAsync(); + producer.newMessage().key("K2").value(null).sendAsync(); + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + producer.newMessage().key("K1").value(null).sendAsync(); + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + Assert.assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + @Cleanup + Reader reader = pulsarClient.newReader(Schema.STRING) + .subscriptionName("reader-test") + .topic(topicName) + .readCompacted(true) + .startMessageId(MessageId.earliest) + .create(); + while (reader.hasMessageAvailable()) { + Message message = reader.readNext(3, TimeUnit.SECONDS); + Assert.assertNotNull(message); + } + // set retain null key back to avoid affecting other tests + pulsar.getConfig().setTopicCompactionRetainNullKey(oldRetainNullKey); + } + @Test public void testCompactAddCompact() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; From fd3d8b9298b13c82bc2bca2149211a865b41ee88 Mon Sep 17 00:00:00 2001 From: Jiwe Guo Date: Mon, 5 Feb 2024 16:30:04 +0800 Subject: [PATCH 450/494] [fix][broker] Fix String wrong format (#21829) --- .../apache/bookkeeper/mledger/impl/ManagedCursorImpl.java | 4 ++-- .../apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java | 2 +- .../pulsar/broker/admin/impl/PersistentTopicsBase.java | 6 +++--- .../broker/service/persistent/PersistentReplicator.java | 2 +- .../broker/service/persistent/PersistentSubscription.java | 2 +- .../buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index fcf1a2a20104f..ef37aec6001f8 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -820,7 +820,7 @@ public void readEntryComplete(Entry entry, Object ctx) { @Override public String toString() { - return String.format("Cursor [{}] get Nth entry", ManagedCursorImpl.this); + return String.format("Cursor [%s] get Nth entry", ManagedCursorImpl.this); } }, null); @@ -1541,7 +1541,7 @@ public synchronized void readEntryFailed(ManagedLedgerException mle, Object ctx) @Override public String toString() { - return String.format("Cursor [{}] async replay entries", ManagedCursorImpl.this); + return String.format("Cursor [%s] async replay entries", ManagedCursorImpl.this); } }; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index f7cd161b9e8a6..a768beacde103 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -1266,7 +1266,7 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { @Override public String toString() { - return String.format("ML [{}] get earliest message publish time of pos", + return String.format("ML [%s] get earliest message publish time of pos", ManagedLedgerImpl.this.name); } }, null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 780537b9cb06b..62493fdafa96a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -2829,7 +2829,7 @@ public void readEntryComplete(Entry entry, Object ctx) { @Override public String toString() { - return String.format("Topic [{}] get entry batch size", + return String.format("Topic [%s] get entry batch size", PersistentTopicsBase.this.topicName); } }, null); @@ -2936,7 +2936,7 @@ public void readEntryComplete(Entry entry, Object ctx) { @Override public String toString() { - return String.format("Topic [{}] internal get message by id", + return String.format("Topic [%s] internal get message by id", PersistentTopicsBase.this.topicName); } }, null); @@ -3108,7 +3108,7 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { @Override public String toString() { - return String.format("Topic [{}] internal examine message async", + return String.format("Topic [%s] internal examine message async", PersistentTopicsBase.this.topicName); } }, null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java index 0b3415243399d..0c3c9d63ad11b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentReplicator.java @@ -532,7 +532,7 @@ public void readEntryComplete(Entry entry, Object ctx) { @Override public String toString() { - return String.format("Replication [{}] peek Nth message", + return String.format("Replication [%s] peek Nth message", PersistentReplicator.this.producer.getProducerName()); } }, null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 973093bb0a8b8..b4c089a227ee7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -860,7 +860,7 @@ public void readEntryComplete(Entry entry, Object ctx) { @Override public String toString() { - return String.format("Subscription [{}-{}] async replay entries", PersistentSubscription.this.topicName, + return String.format("Subscription [%s-%s] async replay entries", PersistentSubscription.this.topicName, PersistentSubscription.this.subName); } }, null); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java index 258a253c12e62..4f85270be95c3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/SnapshotSegmentAbortedTxnProcessorImpl.java @@ -325,7 +325,7 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) { @Override public String toString() { - return String.format("Transaction buffer [{}] recover from snapshot", + return String.format("Transaction buffer [%s] recover from snapshot", SnapshotSegmentAbortedTxnProcessorImpl.this.topic.getName()); } }, null); From abfc10b6d7dce07defb7e295f6cb96124f3288b7 Mon Sep 17 00:00:00 2001 From: lifepuzzlefun Date: Thu, 21 Dec 2023 18:24:08 +0800 Subject: [PATCH 451/494] [improve][broker] Avoid record inactiveproducers when deduplication is disable. (#21193) Co-authored-by: Jiwe Guo --- .../persistent/MessageDeduplication.java | 21 ++++++++ .../persistent/MessageDuplicationTest.java | 49 ++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index 238dc740509b1..802dd91796127 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -474,6 +474,10 @@ private boolean isDeduplicationEnabled() { * Topic will call this method whenever a producer connects. */ public void producerAdded(String producerName) { + if (!isEnabled()) { + return; + } + // Producer is no-longer inactive inactiveProducers.remove(producerName); } @@ -482,6 +486,10 @@ public void producerAdded(String producerName) { * Topic will call this method whenever a producer disconnects. */ public void producerRemoved(String producerName) { + if (!isEnabled()) { + return; + } + // Producer is no-longer active inactiveProducers.put(producerName, System.currentTimeMillis()); } @@ -493,6 +501,14 @@ public synchronized void purgeInactiveProducers() { long minimumActiveTimestamp = System.currentTimeMillis() - TimeUnit.MINUTES .toMillis(pulsar.getConfiguration().getBrokerDeduplicationProducerInactivityTimeoutMinutes()); + // if not enabled just clear all inactive producer record. + if (!isEnabled()) { + if (!inactiveProducers.isEmpty()) { + inactiveProducers.clear(); + } + return; + } + Iterator> mapIterator = inactiveProducers.entrySet().iterator(); boolean hasInactive = false; while (mapIterator.hasNext()) { @@ -545,5 +561,10 @@ ManagedCursor getManagedCursor() { return managedCursor; } + @VisibleForTesting + Map getInactiveProducers() { + return inactiveProducers; + } + private static final Logger log = LoggerFactory.getLogger(MessageDeduplication.class); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java index 19583a4455ead..72eb3e8101fde 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/MessageDuplicationTest.java @@ -32,6 +32,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import io.netty.buffer.ByteBuf; import io.netty.channel.EventLoopGroup; import java.lang.reflect.Field; @@ -47,15 +48,22 @@ import org.apache.pulsar.broker.resources.PulsarResources; import org.apache.pulsar.broker.service.BacklogQuotaManager; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.api.proto.MessageMetadata; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; +import org.awaitility.Awaitility; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Slf4j @Test(groups = "broker") -public class MessageDuplicationTest { +public class MessageDuplicationTest extends BrokerTestBase { private static final int BROKER_DEDUPLICATION_ENTRIES_INTERVAL = 10; private static final int BROKER_DEDUPLICATION_MAX_NUMBER_PRODUCERS = 10; @@ -438,4 +446,43 @@ public void completed(Exception e, long ledgerId, long entryId) { } }); } + + @BeforeMethod(alwaysRun = true) + @Override + protected void setup() throws Exception { + this.conf.setBrokerDeduplicationEnabled(true); + super.baseSetup(); + } + + @AfterMethod(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testMessageDeduplication() throws Exception { + String topicName = "persistent://prop/ns-abc/testMessageDeduplication"; + String producerName = "test-producer"; + Producer producer = pulsarClient + .newProducer(Schema.STRING) + .producerName(producerName) + .topic(topicName) + .create(); + final PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(topicName).get().orElse(null); + assertNotNull(persistentTopic); + final MessageDeduplication messageDeduplication = persistentTopic.getMessageDeduplication(); + assertFalse(messageDeduplication.getInactiveProducers().containsKey(producerName)); + producer.close(); + Awaitility.await().untilAsserted(() -> assertTrue(messageDeduplication.getInactiveProducers().containsKey(producerName))); + admin.topicPolicies().setDeduplicationStatus(topicName, false); + Awaitility.await().untilAsserted(() -> { + final Boolean deduplicationStatus = admin.topicPolicies().getDeduplicationStatus(topicName); + Assert.assertNotNull(deduplicationStatus); + Assert.assertFalse(deduplicationStatus); + }); + messageDeduplication.purgeInactiveProducers(); + assertTrue(messageDeduplication.getInactiveProducers().isEmpty()); + } } From e6cd005f90524222df194a690718f77c4e646670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=93=E5=90=9B?= Date: Fri, 19 Jan 2024 14:38:14 +0800 Subject: [PATCH 452/494] [improve][ml] Filter out deleted entries before read entries from ledger. (#21739) (cherry picked from commit c66167be55e9ed14261174a672952136c6fdb441) --- .../mledger/impl/ManagedCursorImpl.java | 4 + .../mledger/impl/ManagedLedgerImpl.java | 2 +- .../mledger/impl/ReadOnlyCursorImpl.java | 6 + .../mledger/impl/ManagedCursorTest.java | 232 ++++++++++++++++-- 4 files changed, 227 insertions(+), 17 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index ef37aec6001f8..f5ab254476143 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -787,6 +787,8 @@ public void asyncReadEntriesWithSkip(int numberOfEntriesToRead, long maxSizeByte int numOfEntriesToRead = applyMaxSizeCap(numberOfEntriesToRead, maxSizeBytes); PENDING_READ_OPS_UPDATER.incrementAndGet(this); + // Skip deleted entries. + skipCondition = skipCondition == null ? this::isMessageDeleted : skipCondition.or(this::isMessageDeleted); OpReadEntry op = OpReadEntry.create(this, readPosition, numOfEntriesToRead, callback, ctx, maxPosition, skipCondition); ledger.asyncReadEntries(op); @@ -945,6 +947,8 @@ public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, Re asyncReadEntriesWithSkip(numberOfEntriesToRead, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition, skipCondition); } else { + // Skip deleted entries. + skipCondition = skipCondition == null ? this::isMessageDeleted : skipCondition.or(this::isMessageDeleted); OpReadEntry op = OpReadEntry.create(this, readPosition, numberOfEntriesToRead, callback, ctx, maxPosition, skipCondition); diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index a768beacde103..37efc72c32c23 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -4539,4 +4539,4 @@ public Position getTheSlowestNonDurationReadPosition() { } return theSlowestNonDurableReadPosition; } -} +} \ No newline at end of file diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java index 9102339b2904e..1661613f07d7d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorImpl.java @@ -23,6 +23,7 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.ManagedLedgerConfig; +import org.apache.bookkeeper.mledger.Position; import org.apache.bookkeeper.mledger.ReadOnlyCursor; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound; import org.apache.bookkeeper.mledger.proto.MLDataFormats; @@ -70,4 +71,9 @@ public MLDataFormats.ManagedLedgerInfo.LedgerInfo getCurrentLedgerInfo() { public long getNumberOfEntries(Range range) { return this.ledger.getNumberOfEntries(range); } + + @Override + public boolean isMessageDeleted(Position position) { + return false; + } } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java index 627ae73d928bd..644f53c3a522d 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java @@ -43,6 +43,7 @@ import java.util.Arrays; import java.util.BitSet; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -65,6 +66,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Cleanup; @@ -72,6 +74,7 @@ import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.LedgerEntry; +import org.apache.bookkeeper.client.api.ReadHandle; import org.apache.bookkeeper.mledger.AsyncCallbacks; import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback; import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback; @@ -766,7 +769,7 @@ void testResetCursor() throws Exception { @Test(timeOut = 20000) void testResetCursor1() throws Exception { ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", - new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); + new ManagedLedgerConfig().setMaxEntriesPerLedger(2)); ManagedCursor cursor = ledger.openCursor("trc1"); PositionImpl actualEarliest = (PositionImpl) ledger.addEntry("dummy-entry-1".getBytes(Encoding)); ledger.addEntry("dummy-entry-2".getBytes(Encoding)); @@ -2286,7 +2289,7 @@ void testFindNewestMatchingEdgeCase1() throws Exception { ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1"); assertNull(c1.findNewestMatching( - entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding)))); + entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding)))); } @Test(timeOut = 20000) @@ -2595,7 +2598,7 @@ public void findEntryComplete(Position position, Object ctx) { @Override public void findEntryFailed(ManagedLedgerException exception, Optional failedReadPosition, - Object ctx) { + Object ctx) { result.exception = exception; counter.countDown(); } @@ -2621,7 +2624,7 @@ public void findEntryFailed(ManagedLedgerException exception, Optional } void internalTestFindNewestMatchingAllEntries(final String name, final int entriesPerLedger, - final int expectedEntryId) throws Exception { + final int expectedEntryId) throws Exception { final String ledgerAndCursorName = name; ManagedLedgerConfig config = new ManagedLedgerConfig(); config.setRetentionSizeInMB(10); @@ -2715,7 +2718,7 @@ void testReplayEntries() throws Exception { assertTrue((Arrays.equals(entries.get(0).getData(), "entry1".getBytes(Encoding)) && Arrays.equals(entries.get(1).getData(), "entry3".getBytes(Encoding))) || (Arrays.equals(entries.get(0).getData(), "entry3".getBytes(Encoding)) - && Arrays.equals(entries.get(1).getData(), "entry1".getBytes(Encoding)))); + && Arrays.equals(entries.get(1).getData(), "entry1".getBytes(Encoding)))); entries.forEach(Entry::release); // 3. Fail on reading non-existing position @@ -3142,7 +3145,7 @@ public void operationFailed(ManagedLedgerException exception) { try { bkc.openLedgerNoRecovery(ledgerId, DigestType.fromApiDigestType(mlConfig.getDigestType()), - mlConfig.getPassword()); + mlConfig.getPassword()); fail("ledger should have deleted due to update-cursor failure"); } catch (BKException e) { // ok @@ -3761,17 +3764,17 @@ private void deleteBatchIndex(ManagedCursor cursor, Position position, int batch pos.ackSet = bitSet.toLongArray(); cursor.asyncDelete(pos, - new DeleteCallback() { - @Override - public void deleteComplete(Object ctx) { - latch.countDown(); - } + new DeleteCallback() { + @Override + public void deleteComplete(Object ctx) { + latch.countDown(); + } - @Override - public void deleteFailed(ManagedLedgerException exception, Object ctx) { - latch.countDown(); - } - }, null); + @Override + public void deleteFailed(ManagedLedgerException exception, Object ctx) { + latch.countDown(); + } + }, null); latch.await(); pos.ackSet = null; } @@ -4484,5 +4487,202 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { ledger.close(); } + + @Test + public void testReadEntriesWithSkipDeletedEntries() throws Exception { + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testReadEntriesWithSkipDeletedEntries"); + ledger = Mockito.spy(ledger); + List actualReadEntryIds = new ArrayList<>(); + Mockito.doAnswer(inv -> { + long start = inv.getArgument(1); + long end = inv.getArgument(2); + for (long i = start; i <= end; i++) { + actualReadEntryIds.add(i); + } + return inv.callRealMethod(); + }) + .when(ledger) + .asyncReadEntry(Mockito.any(ReadHandle.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + @Cleanup + ManagedCursor cursor = ledger.openCursor("c"); + + int entries = 20; + Position maxReadPosition = null; + Map map = new HashMap<>(); + for (int i = 0; i < entries; i++) { + maxReadPosition = ledger.addEntry(new byte[1024]); + map.put(i, maxReadPosition); + } + + + Set deletedPositions = new HashSet<>(); + deletedPositions.add(map.get(1)); + deletedPositions.add(map.get(4)); + deletedPositions.add(map.get(5)); + deletedPositions.add(map.get(8)); + deletedPositions.add(map.get(9)); + deletedPositions.add(map.get(10)); + deletedPositions.add(map.get(15)); + deletedPositions.add(map.get(17)); + deletedPositions.add(map.get(19)); + cursor.delete(deletedPositions); + + CompletableFuture f0 = new CompletableFuture<>(); + List readEntries = new ArrayList<>(); + cursor.asyncReadEntries(5, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f0.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f0.completeExceptionally(exception); + } + }, null, PositionImpl.get(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + f0.get(); + + CompletableFuture f1 = new CompletableFuture<>(); + cursor.asyncReadEntries(5, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f1.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f1.completeExceptionally(exception); + } + }, null, PositionImpl.get(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + + f1.get(); + CompletableFuture f2 = new CompletableFuture<>(); + cursor.asyncReadEntries(100, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f2.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f2.completeExceptionally(exception); + } + }, null, PositionImpl.get(maxReadPosition.getLedgerId(), maxReadPosition.getEntryId()).getNext()); + + f2.get(); + + Position cursorReadPosition = cursor.getReadPosition(); + Position expectReadPosition = maxReadPosition.getNext(); + assertTrue(cursorReadPosition.getLedgerId() == expectReadPosition.getLedgerId() + && cursorReadPosition.getEntryId() == expectReadPosition.getEntryId()); + + assertEquals(readEntries.size(), actualReadEntryIds.size()); + assertEquals(entries - deletedPositions.size(), actualReadEntryIds.size()); + for (Entry entry : readEntries) { + long entryId = entry.getEntryId(); + assertTrue(actualReadEntryIds.contains(entryId)); + } + } + + + @Test + public void testReadEntriesWithSkipDeletedEntriesAndWithSkipConditions() throws Exception { + @Cleanup + ManagedLedgerImpl ledger = (ManagedLedgerImpl) + factory.open("testReadEntriesWithSkipDeletedEntriesAndWithSkipConditions"); + ledger = Mockito.spy(ledger); + + List actualReadEntryIds = new ArrayList<>(); + Mockito.doAnswer(inv -> { + long start = inv.getArgument(1); + long end = inv.getArgument(2); + for (long i = start; i <= end; i++) { + actualReadEntryIds.add(i); + } + return inv.callRealMethod(); + }) + .when(ledger) + .asyncReadEntry(Mockito.any(ReadHandle.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + @Cleanup + ManagedCursor cursor = ledger.openCursor("c"); + + int entries = 20; + Position maxReadPosition0 = null; + Map map = new HashMap<>(); + for (int i = 0; i < entries; i++) { + maxReadPosition0 = ledger.addEntry(new byte[1024]); + map.put(i, maxReadPosition0); + } + + PositionImpl maxReadPosition = + PositionImpl.get(maxReadPosition0.getLedgerId(), maxReadPosition0.getEntryId()).getNext(); + + Set deletedPositions = new HashSet<>(); + deletedPositions.add(map.get(1)); + deletedPositions.add(map.get(3)); + deletedPositions.add(map.get(5)); + cursor.delete(deletedPositions); + + Set skippedPositions = new HashSet<>(); + skippedPositions.add(map.get(6).getEntryId()); + skippedPositions.add(map.get(7).getEntryId()); + skippedPositions.add(map.get(8).getEntryId()); + skippedPositions.add(map.get(11).getEntryId()); + skippedPositions.add(map.get(15).getEntryId()); + skippedPositions.add(map.get(16).getEntryId()); + + Predicate skipCondition = position -> skippedPositions.contains(position.getEntryId()); + List readEntries = new ArrayList<>(); + + CompletableFuture f0 = new CompletableFuture<>(); + cursor.asyncReadEntriesWithSkip(10, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f0.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f0.completeExceptionally(exception); + } + }, null, maxReadPosition, skipCondition); + + f0.get(); + CompletableFuture f1 = new CompletableFuture<>(); + cursor.asyncReadEntriesWithSkip(100, -1L, new ReadEntriesCallback() { + @Override + public void readEntriesComplete(List entries, Object ctx) { + readEntries.addAll(entries); + f1.complete(null); + } + + @Override + public void readEntriesFailed(ManagedLedgerException exception, Object ctx) { + f1.completeExceptionally(exception); + } + }, null, maxReadPosition, skipCondition); + f1.get(); + + + assertEquals(actualReadEntryIds.size(), readEntries.size()); + assertEquals(entries - deletedPositions.size() - skippedPositions.size(), actualReadEntryIds.size()); + for (Entry entry : readEntries) { + long entryId = entry.getEntryId(); + assertTrue(actualReadEntryIds.contains(entryId)); + } + + Position cursorReadPosition = cursor.getReadPosition(); + Position expectReadPosition = maxReadPosition; + assertTrue(cursorReadPosition.getLedgerId() == expectReadPosition.getLedgerId() + && cursorReadPosition.getEntryId() == expectReadPosition.getEntryId()); + } + private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class); } From 890be166069467d43a708475fef8ba3719cde847 Mon Sep 17 00:00:00 2001 From: Penghui Li Date: Thu, 8 Feb 2024 23:47:59 +0800 Subject: [PATCH 453/494] [fix] [ci] [branch-3.0] Fix the build issue from cherry-pick (#22042) --- .../apache/pulsar/broker/service/OneWayReplicatorTest.java | 1 - .../main/java/org/apache/pulsar/client/impl/ConsumerBase.java | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 61105ebbac5d2..e76fd823a4b16 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -37,7 +37,6 @@ import org.junit.Assert; import org.awaitility.Awaitility; import org.mockito.Mockito; -import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index e933005f2d6ea..fec428824c205 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -193,6 +193,10 @@ protected ConsumerBase(PulsarClientImpl client, String topic, ConsumerConfigurat initReceiverQueueSize(); } + protected UnAckedMessageTracker getUnAckedMessageTracker() { + return unAckedMessageTracker; + } + protected void triggerBatchReceiveTimeoutTask() { if (!hasBatchReceiveTimeout() && batchReceivePolicy.getTimeoutMs() > 0) { batchReceiveTimeout = client.timer().newTimeout(this::pendingBatchReceiveTask, From 04c2da4f20fa14ecb42671131993bd1e76c98fdd Mon Sep 17 00:00:00 2001 From: Jiwe Guo Date: Wed, 27 Dec 2023 21:56:45 +0800 Subject: [PATCH 454/494] Fix testNoCleanupOffloadLedgerWhenMetadataExceptionHappens (cherry picked from commit d276550533115a1030eb46ef69826a8807f191e0) --- .../org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 663ed1015f4c5..6c4f21c3af29d 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -4210,7 +4210,7 @@ public void testNoCleanupOffloadLedgerWhenMetadataExceptionHappens() throws Exce // mock the read handle to make the offload successful CompletableFuture readHandle = new CompletableFuture<>(); readHandle.complete(mock(ReadHandle.class)); - when(ml.getLedgerHandle(eq(ledgerInfo.getLedgerId()))).thenReturn(readHandle); + doReturn(readHandle).when(ml).getLedgerHandle(eq(ledgerInfo.getLedgerId())); when(ledgerOffloader.offload(any(), any(), anyMap())).thenReturn(CompletableFuture.completedFuture(null)); ml.ledgers.put(ledgerInfo.getLedgerId(), ledgerInfo); From a88f592a75964f6d899dd517cd01d29a653d4abe Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 8 Feb 2024 21:43:26 -0800 Subject: [PATCH 455/494] [fix][broker] Sanitize values before logging in apply-config-from-env.py script (#22044) (cherry picked from commit 303678364eab538c16041214cae1588a5b2111d9) --- .../apply-config-from-env-with-prefix.py | 85 ++----------------- .../pulsar/scripts/apply-config-from-env.py | 57 ++++++------- 2 files changed, 32 insertions(+), 110 deletions(-) diff --git a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py index 58f6c98975005..9943b283a9f89 100755 --- a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py +++ b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -32,83 +32,8 @@ # update if they exist and ignored if they don't. ############################################################ -import os -import sys - -if len(sys.argv) < 3: - print('Usage: %s [...]' % (sys.argv[0])) - sys.exit(1) - -# Always apply env config to env scripts as well -prefix = sys.argv[1] -conf_files = sys.argv[2:] - -PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1') - -for conf_filename in conf_files: - lines = [] # List of config file lines - keys = {} # Map a key to its line number in the file - - # Load conf file - for line in open(conf_filename): - lines.append(line) - line = line.strip() - if not line or line.startswith('#'): - continue - - try: - k,v = line.split('=', 1) - keys[k] = len(lines) - 1 - except: - if PF_ENV_DEBUG: - print("[%s] skip Processing %s" % (conf_filename, line)) - - # Update values from Env - for k in sorted(os.environ.keys()): - v = os.environ[k].strip() - - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - if k.startswith(prefix): - k = k[len(prefix):] - if k in keys: - print('[%s] Applying config %s = %s' % (conf_filename, k, displayValue)) - idx = keys[k] - lines[idx] = '%s=%s\n' % (k, v) - - - # Ensure we have a new-line at the end of the file, to avoid issue - # when appending more lines to the config - lines.append('\n') - - # Add new keys from Env - for k in sorted(os.environ.keys()): - v = os.environ[k] - if not k.startswith(prefix): - continue - - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - k = k[len(prefix):] - if k not in keys: - print('[%s] Adding config %s = %s' % (conf_filename, k, displayValue)) - lines.append('%s=%s\n' % (k, v)) - else: - print('[%s] Updating config %s = %s' % (conf_filename, k, displayValue)) - lines[keys[k]] = '%s=%s\n' % (k, v) - - - # Store back the updated config in the same file - f = open(conf_filename, 'w') - for line in lines: - f.write(line) - f.close() +# DEPRECATED: Use "apply-config-from-env.py --prefix MY_PREFIX_ conf_file" instead +# this is not a python script, but a bash script. Call apply-config-from-env.py with the prefix argument +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +"${SCRIPT_DIR}/apply-config-from-env.py" --prefix "$1" "${@:2}" diff --git a/docker/pulsar/scripts/apply-config-from-env.py b/docker/pulsar/scripts/apply-config-from-env.py index b8b479fc15b85..da51f05f8be66 100755 --- a/docker/pulsar/scripts/apply-config-from-env.py +++ b/docker/pulsar/scripts/apply-config-from-env.py @@ -25,18 +25,29 @@ ## ./apply-config-from-env file.conf ## -import os, sys - -if len(sys.argv) < 2: - print('Usage: %s' % (sys.argv[0])) +import os, sys, argparse + +parser = argparse.ArgumentParser(description='Pulsar configuration file customizer based on environment variables') +parser.add_argument('--prefix', default='PULSAR_PREFIX_', help='Prefix for environment variables, default is PULSAR_PREFIX_') +parser.add_argument('conf_files', nargs='*', help='Configuration files') +args = parser.parse_args() +if not args.conf_files: + parser.print_help() sys.exit(1) -# Always apply env config to env scripts as well -conf_files = sys.argv[1:] +env_prefix = args.prefix +conf_files = args.conf_files -PF_ENV_PREFIX = 'PULSAR_PREFIX_' PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1') +# List of keys where the value should not be displayed in logs +sensitive_keys = ["brokerClientAuthenticationParameters", "bookkeeperClientAuthenticationParameters", "tokenSecretKey"] + +def sanitize_display_value(k, v): + if "password" in k.lower() or k in sensitive_keys or (k == "tokenSecretKey" and v.startswith("data:")): + return "********" + return v + for conf_filename in conf_files: lines = [] # List of config file lines keys = {} # Map a key to its line number in the file @@ -47,7 +58,6 @@ line = line.strip() if not line: continue - try: k,v = line.split('=', 1) if k.startswith('#'): @@ -61,37 +71,26 @@ for k in sorted(os.environ.keys()): v = os.environ[k].strip() - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v - - if k.startswith(PF_ENV_PREFIX): - k = k[len(PF_ENV_PREFIX):] if k in keys: + displayValue = sanitize_display_value(k, v) print('[%s] Applying config %s = %s' % (conf_filename, k, displayValue)) idx = keys[k] lines[idx] = '%s=%s\n' % (k, v) - # Ensure we have a new-line at the end of the file, to avoid issue # when appending more lines to the config lines.append('\n') - - # Add new keys from Env + + # Add new keys from Env for k in sorted(os.environ.keys()): - v = os.environ[k] - if not k.startswith(PF_ENV_PREFIX): + if not k.startswith(env_prefix): continue - # Hide the value in logs if is password. - if "password" in k.lower(): - displayValue = "********" - else: - displayValue = v + v = os.environ[k].strip() + k = k[len(env_prefix):] + + displayValue = sanitize_display_value(k, v) - k = k[len(PF_ENV_PREFIX):] if k not in keys: print('[%s] Adding config %s = %s' % (conf_filename, k, displayValue)) lines.append('%s=%s\n' % (k, v)) @@ -99,10 +98,8 @@ print('[%s] Updating config %s = %s' % (conf_filename, k, displayValue)) lines[keys[k]] = '%s=%s\n' % (k, v) - # Store back the updated config in the same file f = open(conf_filename, 'w') for line in lines: f.write(line) - f.close() - + f.close() \ No newline at end of file From d02ba2c3100452241da400bbf1dc2956b3fff2b4 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 19 Feb 2024 14:57:18 +0800 Subject: [PATCH 456/494] [fix][broker][branch-3.1] Avoid PublishRateLimiter use an already closed RateLimiter (#22011) --- .../ResourceGroupPublishLimiter.java | 2 +- .../ResourceGroupRateLimiterTest.java | 87 +++++++++++++++++-- .../PrecisTopicPublishRateThrottleTest.java | 75 +++++++++++++++- .../pulsar/common/util/RateLimiter.java | 7 +- 4 files changed, 161 insertions(+), 10 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupPublishLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupPublishLimiter.java index 85e00bb2f87dc..0377ec86488d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupPublishLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupPublishLimiter.java @@ -135,7 +135,7 @@ public void update(long publishRateInMsgs, long publishRateInBytes) { public boolean tryAcquire(int numbers, long bytes) { return (publishRateLimiterOnMessage == null || publishRateLimiterOnMessage.tryAcquire(numbers)) - && (publishRateLimiterOnByte == null || publishRateLimiterOnByte.tryAcquire(bytes)); + && (publishRateLimiterOnByte == null || publishRateLimiterOnByte.tryAcquire(bytes)); } public void registerRateLimitFunction(String name, RateLimitFunction func) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java index fed827b1517e6..9efd2c109f355 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupRateLimiterTest.java @@ -21,23 +21,30 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; - +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; +import java.util.function.Function; +import lombok.Cleanup; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.service.BrokerTestBase; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.util.RateLimiter; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - public class ResourceGroupRateLimiterTest extends BrokerTestBase { final String rgName = "testRG"; @@ -147,6 +154,76 @@ public void testResourceGroupPublishRateLimit() throws Exception { testRateLimit(); } + @Test + public void testWithConcurrentUpdate() throws Exception { + cleanup(); + setup(); + createResourceGroup(rgName, testAddRg); + admin.namespaces().setNamespaceResourceGroup(namespaceName, rgName); + + Awaitility.await().untilAsserted(() -> + assertNotNull(pulsar.getResourceGroupServiceManager() + .getNamespaceResourceGroup(NamespaceName.get(namespaceName)))); + + Awaitility.await().untilAsserted(() -> + assertNotNull(pulsar.getResourceGroupServiceManager() + .resourceGroupGet(rgName).getResourceGroupPublishLimiter())); + + ResourceGroupPublishLimiter resourceGroupPublishLimiter = Mockito.spy(pulsar.getResourceGroupServiceManager() + .resourceGroupGet(rgName).getResourceGroupPublishLimiter()); + + AtomicBoolean blocking = new AtomicBoolean(false); + BiFunction, Long, Boolean> blockFunc = (function, acquirePermit) -> { + blocking.set(true); + while (blocking.get()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return function.apply(acquirePermit); + }; + + Mockito.doAnswer(invocation -> { + RateLimiter publishRateLimiterOnMessage = + (RateLimiter) FieldUtils.readDeclaredField(resourceGroupPublishLimiter, + "publishRateLimiterOnMessage", true); + RateLimiter publishRateLimiterOnByte = + (RateLimiter) FieldUtils.readDeclaredField(resourceGroupPublishLimiter, + "publishRateLimiterOnByte", true); + int numbers = invocation.getArgument(0); + long bytes = invocation.getArgument(1); + return (publishRateLimiterOnMessage == null || publishRateLimiterOnMessage.tryAcquire(numbers)) + && (publishRateLimiterOnByte == null || blockFunc.apply(publishRateLimiterOnByte::tryAcquire, bytes)); + }).when(resourceGroupPublishLimiter).tryAcquire(Mockito.anyInt(), Mockito.anyLong()); + + ConcurrentHashMap resourceGroupsMap = + (ConcurrentHashMap) FieldUtils.readDeclaredField(pulsar.getResourceGroupServiceManager(), + "resourceGroupsMap", true); + FieldUtils.writeDeclaredField(resourceGroupsMap.get(rgName), "resourceGroupPublishLimiter", + resourceGroupPublishLimiter, true); + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(namespaceName + "/test-topic") + .create(); + + CompletableFuture sendFuture = producer.sendAsync(new byte[MESSAGE_SIZE]); + + Awaitility.await().untilAsserted(() -> Assert.assertTrue(blocking.get())); + + testAddRg.setPublishRateInBytes(Long.valueOf(MESSAGE_SIZE) + 1); + admin.resourcegroups().updateResourceGroup(rgName, testAddRg); + blocking.set(false); + + sendFuture.join(); + + // Now detach the namespace + admin.namespaces().removeNamespaceResourceGroup(namespaceName); + deleteResourceGroup(rgName); + } + + private void prepareData() { testAddRg.setPublishRateInBytes(Long.valueOf(MESSAGE_SIZE)); testAddRg.setPublishRateInMsgs(1); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java index c22ed41fc1533..b13f150387bee 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PrecisTopicPublishRateThrottleTest.java @@ -18,16 +18,25 @@ */ package org.apache.pulsar.broker.service; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; +import java.util.function.Function; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.common.policies.data.PublishRate; +import org.apache.pulsar.common.util.RateLimiter; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - @Test(groups = "broker") +@Slf4j public class PrecisTopicPublishRateThrottleTest extends BrokerTestBase{ @Override @@ -180,4 +189,64 @@ public void testBrokerLevelPublishRateDynamicUpdate() throws Exception{ producer.close(); super.internalCleanup(); } + + @Test + public void testWithConcurrentUpdate() throws Exception { + PublishRate publishRate = new PublishRate(-1,10); + conf.setPreciseTopicPublishRateLimiterEnable(true); + conf.setMaxPendingPublishRequestsPerConnection(1000); + super.baseSetup(); + admin.namespaces().setPublishRate("prop/ns-abc", publishRate); + final String topic = "persistent://prop/ns-abc/testWithConcurrentUpdate"; + @Cleanup + org.apache.pulsar.client.api.Producer producer = pulsarClient.newProducer() + .topic(topic) + .producerName("producer-name") + .create(); + + AbstractTopic topicRef = (AbstractTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + Assert.assertNotNull(topicRef); + + PublishRateLimiter topicPublishRateLimiter = Mockito.spy(topicRef.getTopicPublishRateLimiter()); + + AtomicBoolean blocking = new AtomicBoolean(false); + BiFunction, Long, Boolean> blockFunc = (function, acquirePermit) -> { + blocking.set(true); + while (blocking.get()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + return function.apply(acquirePermit); + }; + + Mockito.doAnswer(invocation -> { + log.info("tryAcquire: {}, {}", invocation.getArgument(0), invocation.getArgument(1)); + RateLimiter publishRateLimiterOnMessage = + (RateLimiter) FieldUtils.readDeclaredField(topicPublishRateLimiter, + "topicPublishRateLimiterOnMessage", true); + RateLimiter publishRateLimiterOnByte = + (RateLimiter) FieldUtils.readDeclaredField(topicPublishRateLimiter, + "topicPublishRateLimiterOnByte", true); + int numbers = invocation.getArgument(0); + long bytes = invocation.getArgument(1); + return (publishRateLimiterOnMessage == null || publishRateLimiterOnMessage.tryAcquire(numbers)) + && (publishRateLimiterOnByte == null || blockFunc.apply(publishRateLimiterOnByte::tryAcquire, bytes)); + }).when(topicPublishRateLimiter).tryAcquire(Mockito.anyInt(), Mockito.anyLong()); + + FieldUtils.writeField(topicRef, "topicPublishRateLimiter", topicPublishRateLimiter, true); + + CompletableFuture sendFuture = producer.sendAsync(new byte[10]); + + Awaitility.await().untilAsserted(() -> Assert.assertTrue(blocking.get())); + publishRate.publishThrottlingRateInByte = 20; + admin.namespaces().setPublishRate("prop/ns-abc", publishRate); + blocking.set(false); + + sendFuture.join(); + + super.internalCleanup(); + } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java index 4ecb29b2462cc..17199664f12d1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimiter.java @@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import lombok.Builder; +import lombok.extern.slf4j.Slf4j; /** * A Rate Limiter that distributes permits at a configurable rate. Each {@link #acquire()} blocks if necessary until a @@ -50,6 +51,7 @@ *

  • Faster: RateLimiter is light-weight and faster than Guava-RateLimiter
    • * */ +@Slf4j public class RateLimiter implements AutoCloseable{ private final ScheduledExecutorService executorService; private long rateTime; @@ -175,7 +177,10 @@ public synchronized boolean tryAcquire() { * @return {@code true} if the permits were acquired, {@code false} otherwise */ public synchronized boolean tryAcquire(long acquirePermit) { - checkArgument(!isClosed(), "Rate limiter is already shutdown"); + if (isClosed()) { + log.info("The current rate limiter is already shutdown, acquire permits directly."); + return true; + } // lazy init and start task only once application start using it if (renewTask == null) { renewTask = createTask(); From 47972afb00696e3cfc9d9fb0465bbb6d3a614154 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 21 Feb 2024 09:34:29 +0200 Subject: [PATCH 457/494] [fix][broker] Support running docker container with gid != 0 (#22081) (cherry picked from commit 4097ddd5e8c4fae4d95c939222341e5ad5dd6d20) --- docker/pulsar/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index a127af166eb52..b465bbf251da8 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -36,10 +36,14 @@ COPY scripts/install-pulsar-client.sh /pulsar/bin # The final image needs to give the root group sufficient permission for Pulsar components # to write to specific directories within /pulsar +# The ownership is changed to uid 10000 to allow using a different root group. This is necessary when running the +# container when gid=0 is prohibited. In that case, the container must be run with uid 10000 with +# any group id != 0 (for example 10001). # The file permissions are preserved when copying files from this builder image to the target image. RUN for SUBDIRECTORY in conf data download logs; do \ [ -d /pulsar/$SUBDIRECTORY ] || mkdir /pulsar/$SUBDIRECTORY; \ - chmod -R g+w /pulsar/$SUBDIRECTORY; \ + chmod -R ug+w /pulsar/$SUBDIRECTORY; \ + chown -R 10000:0 /pulsar/$SUBDIRECTORY; \ done # Trino writes logs to this directory (at least during tests), so we need to give the process permission From 179016b8c5a15836c887c061ae6c8fe05312be45 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Wed, 21 Feb 2024 18:51:56 +0900 Subject: [PATCH 458/494] [fix][sec] Upgrade commons-compress to 1.26.0 (#22086) (cherry picked from commit 613a77100226628d8685d34260685d2df2b405ae) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- .../pulsar/functions/instance/JavaInstanceDepsTest.java | 4 ++++ pulsar-sql/presto-distribution/LICENSE | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 040ccb723ec7d..77d7be44478db 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -285,7 +285,7 @@ The Apache Software License, Version 2.0 - commons-lang-commons-lang-2.6.jar - commons-logging-commons-logging-1.1.1.jar - org.apache.commons-commons-collections4-4.4.jar - - org.apache.commons-commons-compress-1.21.jar + - org.apache.commons-commons-compress-1.26.0.jar - org.apache.commons-commons-lang3-3.11.jar - org.apache.commons-commons-text-1.10.0.jar * Netty diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index c3eedd70fa7f4..cd230e3b0522c 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -346,7 +346,7 @@ The Apache Software License, Version 2.0 - commons-logging-1.2.jar - commons-lang3-3.11.jar - commons-text-1.10.0.jar - - commons-compress-1.21.jar + - commons-compress-1.26.0.jar * Netty - netty-buffer-4.1.100.Final.jar - netty-codec-4.1.100.Final.jar diff --git a/pom.xml b/pom.xml index 08adf869b9875..180a0ac59d311 100644 --- a/pom.xml +++ b/pom.xml @@ -131,7 +131,7 @@ flexible messaging model and an intuitive client API. package - 1.21 + 1.26.0 4.16.4 3.9.1 diff --git a/pulsar-functions/runtime-all/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceDepsTest.java b/pulsar-functions/runtime-all/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceDepsTest.java index 854c146893243..b65bf17f70b44 100644 --- a/pulsar-functions/runtime-all/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceDepsTest.java +++ b/pulsar-functions/runtime-all/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceDepsTest.java @@ -46,6 +46,8 @@ * 8. Apache AVRO * 9. Jackson Mapper and Databind (dependency of AVRO) * 10. Apache Commons Compress (dependency of AVRO) + * 11. Apache Commons Lang (dependency of Apache Commons Compress) + * 12. Apache Commons IO (dependency of Apache Commons Compress) */ public class JavaInstanceDepsTest { @@ -71,6 +73,8 @@ public void testInstanceJarDeps() throws IOException { && !name.startsWith("org/apache/avro") && !name.startsWith("com/fasterxml/jackson") && !name.startsWith("org/apache/commons/compress") + && !name.startsWith("org/apache/commons/lang3") + && !name.startsWith("org/apache/commons/io") && !name.startsWith("com/google") && !name.startsWith("org/checkerframework") && !name.startsWith("javax/annotation") diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 398562c7dc22f..f25ca8ec78bda 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -228,7 +228,7 @@ The Apache Software License, Version 2.0 - guice-5.1.0.jar * Apache Commons - commons-math3-3.6.1.jar - - commons-compress-1.21.jar + - commons-compress-1.26.0.jar - commons-lang3-3.11.jar * Netty - netty-buffer-4.1.100.Final.jar From ff17d99cc85e87dfb7206975b05a1f92c18fda85 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 15 Feb 2024 11:07:10 +0200 Subject: [PATCH 459/494] [fix][broker] Fix hash collision when using a consumer name that ends with a number (#22053) --- ...stentHashingStickyKeyConsumerSelector.java | 14 ++-- ...tHashingStickyKeyConsumerSelectorTest.java | 74 +++++++++++++++---- ...ckyKeyDispatcherMultipleConsumersTest.java | 4 +- 3 files changed, 70 insertions(+), 22 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java index ea491bd40d332..b2b2b512c8cfc 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelector.java @@ -39,7 +39,8 @@ * number of keys assigned to each consumer. */ public class ConsistentHashingStickyKeyConsumerSelector implements StickyKeyConsumerSelector { - + // use NUL character as field separator for hash key calculation + private static final String KEY_SEPARATOR = "\0"; private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); // Consistent-Hash ring @@ -59,8 +60,7 @@ public CompletableFuture addConsumer(Consumer consumer) { // Insert multiple points on the hash ring for every consumer // The points are deterministically added based on the hash of the consumer name for (int i = 0; i < numberOfPoints; i++) { - String key = consumer.consumerName() + i; - int hash = Murmur3_32Hash.getInstance().makeHash(key.getBytes()); + int hash = calculateHashForConsumerAndIndex(consumer, i); hashRing.compute(hash, (k, v) -> { if (v == null) { return Lists.newArrayList(consumer); @@ -79,14 +79,18 @@ public CompletableFuture addConsumer(Consumer consumer) { } } + private static int calculateHashForConsumerAndIndex(Consumer consumer, int index) { + String key = consumer.consumerName() + KEY_SEPARATOR + index; + return Murmur3_32Hash.getInstance().makeHash(key.getBytes()); + } + @Override public void removeConsumer(Consumer consumer) { rwLock.writeLock().lock(); try { // Remove all the points that were added for this consumer for (int i = 0; i < numberOfPoints; i++) { - String key = consumer.consumerName() + i; - int hash = Murmur3_32Hash.getInstance().makeHash(key.getBytes()); + int hash = calculateHashForConsumerAndIndex(consumer, i); hashRing.compute(hash, (k, v) -> { if (v == null) { return null; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java index dbca31416bb9d..48311c57338b5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ConsistentHashingStickyKeyConsumerSelectorTest.java @@ -21,18 +21,18 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; - -import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; -import org.apache.pulsar.client.api.Range; -import org.testng.Assert; -import org.testng.annotations.Test; - import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.pulsar.broker.service.BrokerServiceException.ConsumerAssignException; +import org.apache.pulsar.client.api.Range; +import org.testng.Assert; +import org.testng.annotations.Test; @Test(groups = "broker") public class ConsistentHashingStickyKeyConsumerSelectorTest { @@ -154,17 +154,17 @@ public void testGetConsumerKeyHashRanges() throws BrokerServiceException.Consume } Map> expectedResult = new HashMap<>(); expectedResult.put(consumers.get(0), Arrays.asList( - Range.of(0, 330121749), - Range.of(330121750, 618146114), - Range.of(1797637922, 1976098885))); + Range.of(119056335, 242013991), + Range.of(722195657, 1656011842), + Range.of(1707482098, 1914695766))); expectedResult.put(consumers.get(1), Arrays.asList( - Range.of(938427576, 1094135919), - Range.of(1138613629, 1342907082), - Range.of(1342907083, 1797637921))); + Range.of(0, 90164503), + Range.of(90164504, 119056334), + Range.of(382436668, 722195656))); expectedResult.put(consumers.get(2), Arrays.asList( - Range.of(618146115, 772640562), - Range.of(772640563, 938427575), - Range.of(1094135920, 1138613628))); + Range.of(242013992, 242377547), + Range.of(242377548, 382436667), + Range.of(1656011843, 1707482097))); for (Map.Entry> entry : selector.getConsumerKeyHashRanges().entrySet()) { System.out.println(entry.getValue()); Assert.assertEquals(entry.getValue(), expectedResult.get(entry.getKey())); @@ -172,4 +172,48 @@ public void testGetConsumerKeyHashRanges() throws BrokerServiceException.Consume } Assert.assertEquals(expectedResult.size(), 0); } + + // reproduces https://github.com/apache/pulsar/issues/22050 + @Test + public void shouldNotCollideWithConsumerNameEndsWithNumber() { + ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(12); + List consumerName = Arrays.asList("consumer1", "consumer11"); + List consumers = new ArrayList<>(); + for (String s : consumerName) { + Consumer consumer = mock(Consumer.class); + when(consumer.consumerName()).thenReturn(s); + selector.addConsumer(consumer); + consumers.add(consumer); + } + Map rangeToConsumer = new HashMap<>(); + for (Map.Entry> entry : selector.getConsumerKeyHashRanges().entrySet()) { + for (Range range : entry.getValue()) { + Consumer previous = rangeToConsumer.put(range, entry.getKey()); + if (previous != null) { + Assert.fail("Ranges are colliding between " + previous.consumerName() + " and " + entry.getKey() + .consumerName()); + } + } + } + } + + @Test + public void shouldRemoveConsumersFromConsumerKeyHashRanges() { + ConsistentHashingStickyKeyConsumerSelector selector = new ConsistentHashingStickyKeyConsumerSelector(12); + List consumers = IntStream.range(1, 100).mapToObj(i -> "consumer" + i) + .map(consumerName -> { + Consumer consumer = mock(Consumer.class); + when(consumer.consumerName()).thenReturn(consumerName); + return consumer; + }).collect(Collectors.toList()); + + // when consumers are added + consumers.forEach(selector::addConsumer); + // then each consumer should have a range + Assert.assertEquals(selector.getConsumerKeyHashRanges().size(), consumers.size()); + // when consumers are removed + consumers.forEach(selector::removeConsumer); + // then there should be no mapping remaining + Assert.assertEquals(selector.getConsumerKeyHashRanges().size(), 0); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java index 48a4bfc923608..7e1b5f8c71e6d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentStickyKeyDispatcherMultipleConsumersTest.java @@ -250,7 +250,7 @@ public void testSkipRedeliverTemporally() { redeliverEntries.add(EntryImpl.create(1, 1, createMessage("message1", 1, "key1"))); final List readEntries = new ArrayList<>(); readEntries.add(EntryImpl.create(1, 2, createMessage("message2", 2, "key1"))); - readEntries.add(EntryImpl.create(1, 3, createMessage("message3", 3, "key2"))); + readEntries.add(EntryImpl.create(1, 3, createMessage("message3", 3, "key22"))); try { Field totalAvailablePermitsField = PersistentDispatcherMultipleConsumers.class.getDeclaredField("totalAvailablePermits"); @@ -346,7 +346,7 @@ public void testMessageRedelivery() throws Exception { // Messages with key1 are routed to consumer1 and messages with key2 are routed to consumer2 final List allEntries = new ArrayList<>(); - allEntries.add(EntryImpl.create(1, 1, createMessage("message1", 1, "key2"))); + allEntries.add(EntryImpl.create(1, 1, createMessage("message1", 1, "key22"))); allEntries.add(EntryImpl.create(1, 2, createMessage("message2", 2, "key1"))); allEntries.add(EntryImpl.create(1, 3, createMessage("message3", 3, "key1"))); allEntries.forEach(entry -> ((EntryImpl) entry).retain()); From 1790438b829c1391e5f87e2db59392ae3435c366 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 19 Feb 2024 16:43:39 +0800 Subject: [PATCH 460/494] [fix] [broker] Fix can not subscribe partitioned topic with a suffix-matched regexp (#22025) --- .../broker/resources/TopicResources.java | 3 ++ .../broker/namespace/NamespaceService.java | 3 ++ .../service/PulsarCommandSenderImpl.java | 6 ++++ .../broker/service/TopicListService.java | 22 ++++++++++-- .../impl/PatternTopicsConsumerImplTest.java | 34 +++++++++++++++---- .../pulsar/client/api/ConsumerBuilder.java | 8 ++--- .../client/impl/MultiTopicsConsumerImpl.java | 10 ++++-- .../impl/PatternMultiTopicsConsumerImpl.java | 24 +++++++++++-- .../pulsar/client/impl/TopicListWatcher.java | 4 ++- .../impl/conf/ConsumerConfigurationData.java | 2 +- .../pulsar/common/protocol/Commands.java | 7 ++++ 11 files changed, 104 insertions(+), 19 deletions(-) diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java index 840ced0a1c1c4..0963f25c3d31f 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/TopicResources.java @@ -50,6 +50,9 @@ public TopicResources(MetadataStore store) { store.registerListener(this::handleNotification); } + /*** + * List persistent topics names under a namespace, the topic name contains the partition suffix. + */ public CompletableFuture> listPersistentTopicsAsync(NamespaceName ns) { String path = MANAGED_LEDGER_PATH + "/" + ns + "/persistent"; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index 3be14a9d09a8a..e613e0d4c4278 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1413,6 +1413,9 @@ private CompletableFuture> getPartitionsForTopic(TopicName topicNam }); } + /*** + * List persistent topics names under a namespace, the topic name contains the partition suffix. + */ public CompletableFuture> getListOfPersistentTopics(NamespaceName namespaceName) { return pulsar.getPulsarResources().getTopicResources().listPersistentTopicsAsync(namespaceName); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java index dd74fc4e71ed2..105650caaaf13 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarCommandSenderImpl.java @@ -356,12 +356,18 @@ public void sendEndTxnErrorResponse(long requestId, TxnID txnID, ServerError err writeAndFlush(outBuf); } + /*** + * @param topics topic names which are matching, the topic name contains the partition suffix. + */ @Override public void sendWatchTopicListSuccess(long requestId, long watcherId, String topicsHash, List topics) { BaseCommand command = Commands.newWatchTopicListSuccess(requestId, watcherId, topicsHash, topics); interceptAndWriteCommand(command); } + /*** + * {@inheritDoc} + */ @Override public void sendWatchTopicListUpdate(long watcherId, List newTopics, List deletedTopics, String topicsHash) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java index 7aa50057d73c9..aea5b9fc65b46 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/TopicListService.java @@ -31,6 +31,7 @@ import org.apache.pulsar.common.api.proto.CommandWatchTopicListClose; import org.apache.pulsar.common.api.proto.ServerError; import org.apache.pulsar.common.naming.NamespaceName; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.topics.TopicList; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.metadata.api.NotificationType; @@ -42,11 +43,16 @@ public class TopicListService { public static class TopicListWatcher implements BiConsumer { + /** Topic names which are matching, the topic name contains the partition suffix. **/ private final List matchingTopics; private final TopicListService topicListService; private final long id; + /** The regexp for the topic name(not contains partition suffix). **/ private final Pattern topicsPattern; + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public TopicListWatcher(TopicListService topicListService, long id, Pattern topicsPattern, List topics) { this.topicListService = topicListService; @@ -59,9 +65,12 @@ public List getMatchingTopics() { return matchingTopics; } + /*** + * @param topicName topic name which contains partition suffix. + */ @Override public void accept(String topicName, NotificationType notificationType) { - if (topicsPattern.matcher(topicName).matches()) { + if (topicsPattern.matcher(TopicName.get(topicName).getPartitionedTopicName()).matches()) { List newTopics; List deletedTopics; if (notificationType == NotificationType.Deleted) { @@ -109,6 +118,9 @@ public void inactivate() { } } + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, long requestId, Pattern topicsPattern, String topicsHash, Semaphore lookupSemaphore) { @@ -184,7 +196,9 @@ public void handleWatchTopicList(NamespaceName namespaceName, long watcherId, lo }); } - + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public void initializeTopicsListWatcher(CompletableFuture watcherFuture, NamespaceName namespace, long watcherId, Pattern topicsPattern) { namespaceService.getListOfPersistentTopics(namespace). @@ -246,6 +260,10 @@ public void deleteTopicListWatcher(Long watcherId) { log.info("[{}] Closed watcher, watcherId={}", connection.getRemoteAddress(), watcherId); } + /** + * @param deletedTopics topic names deleted(contains the partition suffix). + * @param newTopics topics names added(contains the partition suffix). + */ public void sendTopicListUpdate(long watcherId, String topicsHash, List deletedTopics, List newTopics) { connection.getCommandSender().sendWatchTopicListUpdate(watcherId, newTopics, deletedTopics, topicsHash); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java index 25f9da4c64920..2ae8ef1d03902 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PatternTopicsConsumerImplTest.java @@ -681,16 +681,27 @@ public void testAutoSubscribePatterConsumerFromBrokerWatcher(boolean delayWatchi } } - @DataProvider(name= "partitioned") - public Object[][] partitioned(){ + @DataProvider(name= "regexpConsumerArgs") + public Object[][] regexpConsumerArgs(){ return new Object[][]{ - {true}, - {false} + {true, true}, + {true, false}, + {false, true}, + {false, false} }; } - @Test(timeOut = testTimeout, dataProvider = "partitioned") - public void testPreciseRegexpSubscribe(boolean partitioned) throws Exception { + private void waitForTopicListWatcherStarted(Consumer consumer) { + Awaitility.await().untilAsserted(() -> { + CompletableFuture completableFuture = WhiteboxImpl.getInternalState(consumer, "watcherFuture"); + log.info("isDone: {}, isCompletedExceptionally: {}", completableFuture.isDone(), + completableFuture.isCompletedExceptionally()); + assertTrue(completableFuture.isDone() && !completableFuture.isCompletedExceptionally()); + }); + } + + @Test(timeOut = testTimeout, dataProvider = "regexpConsumerArgs") + public void testPreciseRegexpSubscribe(boolean partitioned, boolean createTopicAfterWatcherStarted) throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); final String subscriptionName = "s1"; final Pattern pattern = Pattern.compile(String.format("%s$", topicName)); @@ -704,6 +715,9 @@ public void testPreciseRegexpSubscribe(boolean partitioned) throws Exception { .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS) .receiverQueueSize(4) .subscribe(); + if (createTopicAfterWatcherStarted) { + waitForTopicListWatcherStarted(consumer); + } // 1. create topic. if (partitioned) { @@ -733,6 +747,14 @@ public void testPreciseRegexpSubscribe(boolean partitioned) throws Exception { } } + @DataProvider(name= "partitioned") + public Object[][] partitioned(){ + return new Object[][]{ + {true}, + {true} + }; + } + @Test(timeOut = 240 * 1000, dataProvider = "partitioned") public void testPreciseRegexpSubscribeDisabledTopicWatcher(boolean partitioned) throws Exception { final String topicName = BrokerTestUtil.newUniqueName("persistent://public/default/tp"); diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java index 14a94cb8286dc..01c9fc4a86477 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ConsumerBuilder.java @@ -126,7 +126,7 @@ public interface ConsumerBuilder extends Cloneable { ConsumerBuilder topics(List topicNames); /** - * Specify a pattern for topics that this consumer subscribes to. + * Specify a pattern for topics(not contains the partition suffix) that this consumer subscribes to. * *

    The pattern is applied to subscribe to all topics, within a single namespace, that match the * pattern. @@ -134,13 +134,13 @@ public interface ConsumerBuilder extends Cloneable { *

    The consumer automatically subscribes to topics created after itself. * * @param topicsPattern - * a regular expression to select a list of topics to subscribe to + * a regular expression to select a list of topics(not contains the partition suffix) to subscribe to * @return the consumer builder instance */ ConsumerBuilder topicsPattern(Pattern topicsPattern); /** - * Specify a pattern for topics that this consumer subscribes to. + * Specify a pattern for topics(not contains the partition suffix) that this consumer subscribes to. * *

    It accepts a regular expression that is compiled into a pattern internally. E.g., * "persistent://public/default/pattern-topic-.*" @@ -151,7 +151,7 @@ public interface ConsumerBuilder extends Cloneable { *

    The consumer automatically subscribes to topics created after itself. * * @param topicsPattern - * given regular expression for topics pattern + * given regular expression for topics(not contains the partition suffix) pattern * @return the consumer builder instance */ ConsumerBuilder topicsPattern(String topicsPattern); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 5e16dcb0f750a..fd65a9371d47b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -926,7 +926,10 @@ private void removeTopic(String topic) { } } - // subscribe one more given topic + /*** + * Subscribe one more given topic. + * @param topicName topic name without the partition suffix. + */ public CompletableFuture subscribeAsync(String topicName, boolean createTopicIfDoesNotExist) { TopicName topicNameInstance = getTopicName(topicName); if (topicNameInstance == null) { @@ -1247,7 +1250,10 @@ public CompletableFuture unsubscribeAsync(String topicName) { return unsubscribeFuture; } - // Remove a consumer for a topic + /*** + * Remove a consumer for a topic. + * @param topicName topic name contains the partition suffix. + */ public CompletableFuture removeConsumerAsync(String topicName) { checkArgument(TopicName.isValid(topicName), "Invalid topic name:" + topicName); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java index f3ebcdee6c0d9..4d179f7d914c2 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PatternMultiTopicsConsumerImpl.java @@ -67,6 +67,9 @@ public class PatternMultiTopicsConsumerImpl extends MultiTopicsConsumerImpl onTopicsRemoved(Collection removedTopics); - // subscribe and create a list of new ConsumerImpl, added them to the `consumers` map in - // `MultiTopicsConsumerImpl`. + + /*** + * subscribe and create a list of new {@link ConsumerImpl}, added them to the + * {@link MultiTopicsConsumerImpl#consumers} map in {@link MultiTopicsConsumerImpl}. + * @param addedTopics topic names added(contains the partition suffix). + */ CompletableFuture onTopicsAdded(Collection addedTopics); } private class PatternTopicsChangedListener implements TopicsChangedListener { + + /** + * {@inheritDoc} + */ @Override public CompletableFuture onTopicsRemoved(Collection removedTopics) { CompletableFuture removeFuture = new CompletableFuture<>(); @@ -249,6 +264,9 @@ public CompletableFuture onTopicsRemoved(Collection removedTopics) return removeFuture; } + /** + * {@inheritDoc} + */ @Override public CompletableFuture onTopicsAdded(Collection addedTopics) { CompletableFuture addFuture = new CompletableFuture<>(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java index 489a07a606eb2..86adf69f06e0f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TopicListWatcher.java @@ -59,6 +59,9 @@ public class TopicListWatcher extends HandlerState implements ConnectionHandler. private final Runnable recheckTopicsChangeAfterReconnect; + /*** + * @param topicsPattern The regexp for the topic name(not contains partition suffix). + */ public TopicListWatcher(PatternMultiTopicsConsumerImpl.TopicsChangedListener topicsChangeListener, PulsarClientImpl client, Pattern topicsPattern, long watcherId, NamespaceName namespace, String topicsHash, @@ -142,7 +145,6 @@ public CompletableFuture connectionOpened(ClientCnx cnx) { return; } } - this.connectionHandler.resetBackoff(); recheckTopicsChangeAfterReconnect.run(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java index 8760926792cd7..3ae0e977d13c4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ConsumerConfigurationData.java @@ -65,7 +65,7 @@ public class ConsumerConfigurationData implements Serializable, Cloneable { @ApiModelProperty( name = "topicsPattern", - value = "Topic pattern" + value = "The regexp for the topic name(not contains partition suffix)." ) private Pattern topicsPattern; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java index cf0cd820a6d10..3982900041813 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/protocol/Commands.java @@ -1555,6 +1555,9 @@ public static BaseCommand newWatchTopicList( return cmd; } + /*** + * @param topics topic names which are matching, the topic name contains the partition suffix. + */ public static BaseCommand newWatchTopicListSuccess(long requestId, long watcherId, String topicsHash, List topics) { BaseCommand cmd = localCmd(Type.WATCH_TOPIC_LIST_SUCCESS); @@ -1570,6 +1573,10 @@ public static BaseCommand newWatchTopicListSuccess(long requestId, long watcherI return cmd; } + /** + * @param deletedTopics topic names deleted(contains the partition suffix). + * @param newTopics topics names added(contains the partition suffix). + */ public static BaseCommand newWatchTopicUpdate(long watcherId, List newTopics, List deletedTopics, String topicsHash) { BaseCommand cmd = localCmd(Type.WATCH_TOPIC_UPDATE); From 6ecccddbf68390d342be4d418e263d0baee21221 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 6 Feb 2024 15:28:43 +0800 Subject: [PATCH 461/494] [improve][broker] Do not retain the data in the system topic (#22022) ### Motivation For some use case, the users need to store all the messages even though these message are acked by all subscription. So they set the retention policy of the namespace to infinite retention (setting both time and size limits to `-1`). But the data in the system topic does not need for infinite retention. ### Modifications For system topics, do not retain messages that have already been acknowledged. --- .../pulsar/broker/service/BrokerService.java | 15 ++++-- .../compaction/CompactionRetentionTest.java | 48 +++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index d9cea76ce9efe..c5775e023bbab 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1850,10 +1850,17 @@ private CompletableFuture getManagedLedgerConfig(@Nonnull T } if (retentionPolicies == null) { - retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( - () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), - serviceConfig.getDefaultRetentionSizeInMB()) - ); + if (SystemTopicNames.isSystemTopic(topicName)) { + if (log.isDebugEnabled()) { + log.debug("{} Disable data retention policy for system topic.", topicName); + } + retentionPolicies = new RetentionPolicies(0, 0); + } else { + retentionPolicies = policies.map(p -> p.retention_policies).orElseGet( + () -> new RetentionPolicies(serviceConfig.getDefaultRetentionTimeInMinutes(), + serviceConfig.getDefaultRetentionSizeInMB()) + ); + } } ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java index 055c595fbfec8..98bf2b819c2ba 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java @@ -38,6 +38,7 @@ import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BookKeeper; +import org.apache.bookkeeper.mledger.ManagedLedgerConfig; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -45,9 +46,13 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.Reader; import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.naming.SystemTopicNames; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.RetentionPolicies; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.awaitility.Awaitility; +import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -212,6 +217,49 @@ public void testCompactionRetentionOnTopicCreationWithTopicPolicies() throws Exc ); } + @Test + public void testRetentionPolicesForSystemTopic() throws Exception { + String namespace = "my-tenant/my-ns"; + String topicPrefix = "persistent://" + namespace + "/"; + admin.namespaces().setRetention(namespace, new RetentionPolicies(-1, -1)); + // Check event topics and transaction internal topics. + for (String eventTopic : SystemTopicNames.EVENTS_TOPIC_NAMES) { + checkSystemTopicRetentionPolicy(topicPrefix + eventTopic); + } + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_COORDINATOR_ASSIGN); + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_COORDINATOR_LOG); + checkSystemTopicRetentionPolicy(topicPrefix + SystemTopicNames.PENDING_ACK_STORE_SUFFIX); + + // Check common topics. + checkCommonTopicRetentionPolicy(topicPrefix + "my-topic" + System.nanoTime()); + // Specify retention policies for system topic. + pulsar.getConfiguration().setTopicLevelPoliciesEnabled(true); + pulsar.getConfiguration().setSystemTopicEnabled(true); + admin.topics().createNonPartitionedTopic(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT); + admin.topicPolicies().setRetention(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, + new RetentionPolicies(10, 10)); + Awaitility.await().untilAsserted(() -> { + checkTopicRetentionPolicy(topicPrefix + SystemTopicNames.TRANSACTION_BUFFER_SNAPSHOT, + new RetentionPolicies(10, 10)); + }); + } + + private void checkSystemTopicRetentionPolicy(String topicName) throws Exception { + checkTopicRetentionPolicy(topicName, new RetentionPolicies(0, 0)); + + } + + private void checkCommonTopicRetentionPolicy(String topicName) throws Exception { + checkTopicRetentionPolicy(topicName, new RetentionPolicies(-1, -1)); + } + + private void checkTopicRetentionPolicy(String topicName, RetentionPolicies retentionPolicies) throws Exception { + ManagedLedgerConfig config = pulsar.getBrokerService() + .getManagedLedgerConfig(TopicName.get(topicName)).get(); + Assert.assertEquals(config.getRetentionSizeInMB(), retentionPolicies.getRetentionSizeInMB()); + Assert.assertEquals(config.getRetentionTimeMillis(),retentionPolicies.getRetentionTimeInMinutes() * 60000L); + } + private void testCompactionCursorRetention(String topic) throws Exception { Set keys = Sets.newHashSet("a", "b", "c"); Set keysToExpire = Sets.newHashSet("x1", "x2"); From 15299273ec0f4e53a848d51b8ec999900590651a Mon Sep 17 00:00:00 2001 From: atomchen <492672043@qq.com> Date: Sun, 18 Feb 2024 15:51:49 +0800 Subject: [PATCH 462/494] =?UTF-8?q?[fix][broker]Support=20setting=20`autoS?= =?UTF-8?q?kipNonRecoverableData`=20dynamically=20in=20expiryMon=E2=80=A6?= =?UTF-8?q?=20(#21991)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: atomchchen --- .../PersistentMessageExpiryMonitor.java | 10 ++++--- .../persistent/PersistentTopicTest.java | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java index adc86e2f6584f..41bc6098e1a2f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentMessageExpiryMonitor.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.service.persistent; +import com.google.common.annotations.VisibleForTesting; import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; @@ -46,7 +47,6 @@ public class PersistentMessageExpiryMonitor implements FindEntryCallback { private final String topicName; private final Rate msgExpired; private final LongAdder totalMsgExpired; - private final boolean autoSkipNonRecoverableData; private final PersistentSubscription subscription; private static final int FALSE = 0; @@ -65,8 +65,12 @@ public PersistentMessageExpiryMonitor(String topicName, String subscriptionName, this.subscription = subscription; this.msgExpired = new Rate(); this.totalMsgExpired = new LongAdder(); + } + + @VisibleForTesting + public boolean isAutoSkipNonRecoverableData() { // check to avoid test failures - this.autoSkipNonRecoverableData = this.cursor.getManagedLedger() != null + return this.cursor.getManagedLedger() != null && this.cursor.getManagedLedger().getConfig().isAutoSkipNonRecoverableData(); } @@ -190,7 +194,7 @@ public void findEntryFailed(ManagedLedgerException exception, Optional if (log.isDebugEnabled()) { log.debug("[{}][{}] Finding expired entry operation failed", topicName, subName, exception); } - if (autoSkipNonRecoverableData && failedReadPosition.isPresent() + if (isAutoSkipNonRecoverableData() && failedReadPosition.isPresent() && (exception instanceof NonRecoverableLedgerException)) { log.warn("[{}][{}] read failed from ledger at position:{} : {}", topicName, subName, failedReadPosition, exception.getMessage()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 5db17d7ecfa5e..4f3e1e930bd51 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -634,4 +634,30 @@ public void testCheckPersistencePolicies() throws Exception { assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionSizeInMB(), 1L); assertEquals(persistentTopic.getManagedLedger().getConfig().getRetentionTimeMillis(), TimeUnit.MINUTES.toMillis(1)); } + + @Test + public void testDynamicConfigurationAutoSkipNonRecoverableData() throws Exception { + pulsar.getConfiguration().setAutoSkipNonRecoverableData(false); + final String topicName = "persistent://prop/ns-abc/testAutoSkipNonRecoverableData"; + final String subName = "test_sub"; + + Consumer subscribe = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe(); + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topicName, false).join().get(); + PersistentSubscription subscription = persistentTopic.getSubscription(subName); + + assertFalse(persistentTopic.ledger.getConfig().isAutoSkipNonRecoverableData()); + assertFalse(subscription.getExpiryMonitor().isAutoSkipNonRecoverableData()); + + String key = "autoSkipNonRecoverableData"; + admin.brokers().updateDynamicConfiguration(key, "true"); + Awaitility.await() + .untilAsserted(() -> assertEquals(admin.brokers().getAllDynamicConfigurations().get(key), "true")); + + assertTrue(persistentTopic.ledger.getConfig().isAutoSkipNonRecoverableData()); + assertTrue(subscription.getExpiryMonitor().isAutoSkipNonRecoverableData()); + + subscribe.close(); + admin.topics().delete(topicName); + } } From 30697bd382da0c5a4458f3a7c71d2c9c64ee6b63 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 19 Feb 2024 00:04:10 +0800 Subject: [PATCH 463/494] [fix] [broker] Subscription stuck due to called Admin API analyzeSubscriptionBacklog (#22019) --- .../mledger/impl/ManagedCursorImpl.java | 29 ++++++++++++++++-- .../persistent/PersistentSubscription.java | 30 ++++++++++++++++--- .../pulsar/broker/admin/AdminApi2Test.java | 29 ++++++++++++++++++ .../admin/AnalyzeBacklogSubscriptionTest.java | 18 +++++------ .../util/collections/BitSetRecyclable.java | 8 +++++ 5 files changed, 99 insertions(+), 15 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index f5ab254476143..d0fdfe50165d3 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -196,11 +196,11 @@ public class ManagedCursorImpl implements ManagedCursor { position.ackSet = null; return position; }; - private final RangeSetWrapper individualDeletedMessages; + protected final RangeSetWrapper individualDeletedMessages; // Maintain the deletion status for batch messages // (ledgerId, entryId) -> deletion indexes - private final ConcurrentSkipListMap batchDeletedIndexes; + protected final ConcurrentSkipListMap batchDeletedIndexes; private final ReadWriteLock lock = new ReentrantReadWriteLock(); private RateLimiter markDeleteLimiter; @@ -3598,4 +3598,29 @@ public boolean isCacheReadEntry() { public ManagedLedgerConfig getConfig() { return config; } + + /*** + * Create a non-durable cursor and copy the ack stats. + */ + public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException { + NonDurableCursorImpl newNonDurableCursor = + (NonDurableCursorImpl) ledger.newNonDurableCursor(getMarkDeletedPosition(), nonDurableCursorName); + if (individualDeletedMessages != null) { + this.individualDeletedMessages.forEach(range -> { + newNonDurableCursor.individualDeletedMessages.addOpenClosed( + range.lowerEndpoint().getLedgerId(), + range.lowerEndpoint().getEntryId(), + range.upperEndpoint().getLedgerId(), + range.upperEndpoint().getEntryId()); + return true; + }); + } + if (batchDeletedIndexes != null) { + for (Map.Entry entry : this.batchDeletedIndexes.entrySet()) { + BitSetRecyclable copiedBitSet = BitSetRecyclable.valueOf(entry.getValue()); + newNonDurableCursor.batchDeletedIndexes.put(entry.getKey(), copiedBitSet); + } + } + return newNonDurableCursor; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index b4c089a227ee7..42db3dad175c1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Optional; import java.util.TreeMap; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; @@ -525,9 +526,15 @@ public String getTypeString() { return "Null"; } - @Override public CompletableFuture analyzeBacklog(Optional position) { - + final ManagedLedger managedLedger = topic.getManagedLedger(); + final String newNonDurableCursorName = "analyze-backlog-" + UUID.randomUUID(); + ManagedCursor newNonDurableCursor; + try { + newNonDurableCursor = ((ManagedCursorImpl) cursor).duplicateNonDurableCursor(newNonDurableCursorName); + } catch (ManagedLedgerException e) { + return CompletableFuture.failedFuture(e); + } long start = System.currentTimeMillis(); if (log.isDebugEnabled()) { log.debug("[{}][{}] Starting to analyze backlog", topicName, subName); @@ -542,7 +549,7 @@ public CompletableFuture analyzeBacklog(Optional AtomicLong rejectedMessages = new AtomicLong(); AtomicLong rescheduledMessages = new AtomicLong(); - Position currentPosition = cursor.getMarkDeletedPosition(); + Position currentPosition = newNonDurableCursor.getMarkDeletedPosition(); if (log.isDebugEnabled()) { log.debug("[{}][{}] currentPosition {}", @@ -602,7 +609,7 @@ public CompletableFuture analyzeBacklog(Optional return true; }; - return cursor.scan( + CompletableFuture res = newNonDurableCursor.scan( position, condition, batchSize, @@ -629,7 +636,22 @@ public CompletableFuture analyzeBacklog(Optional topicName, subName, end - start, result); return result; }); + res.whenComplete((__, ex) -> { + managedLedger.asyncDeleteCursor(newNonDurableCursorName, + new AsyncCallbacks.DeleteCursorCallback(){ + @Override + public void deleteCursorComplete(Object ctx) { + // Nothing to do. + } + @Override + public void deleteCursorFailed(ManagedLedgerException exception, Object ctx) { + log.warn("[{}][{}] Delete non-durable cursor[{}] failed when analyze backlog.", + topicName, subName, newNonDurableCursor.getName()); + } + }, null); + }); + return res; } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 6ce4c24a191ca..11564cdf72188 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -3388,4 +3388,33 @@ private void testSetBacklogQuotasNamespaceLevelIfRetentionExists() throws Except // cleanup. admin.namespaces().deleteNamespace(ns); } + + @Test + private void testAnalyzeSubscriptionBacklogNotCauseStuck() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp"); + final String subscription = "s1"; + admin.topics().createNonPartitionedTopic(topic); + // Send 10 messages. + Consumer consumer = pulsarClient.newConsumer(Schema.STRING).topic(topic).subscriptionName(subscription) + .receiverQueueSize(0).subscribe(); + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).create(); + for (int i = 0; i < 10; i++) { + producer.send(i + ""); + } + + // Verify consumer can receive all messages after calling "analyzeSubscriptionBacklog". + admin.topics().analyzeSubscriptionBacklog(topic, subscription, Optional.of(MessageIdImpl.earliest)); + for (int i = 0; i < 10; i++) { + Awaitility.await().untilAsserted(() -> { + Message m = consumer.receive(); + assertNotNull(m); + consumer.acknowledge(m); + }); + } + + // cleanup. + consumer.close(); + producer.close(); + admin.topics().delete(topic); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java index 64b2a58ab86e8..f8aa3dc355d92 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AnalyzeBacklogSubscriptionTest.java @@ -154,17 +154,17 @@ private void verifyBacklog(String topic, String subscription, int numEntries, in AnalyzeSubscriptionBacklogResult analyzeSubscriptionBacklogResult = admin.topics().analyzeSubscriptionBacklog(topic, subscription, Optional.empty()); - assertEquals(numEntries, analyzeSubscriptionBacklogResult.getEntries()); - assertEquals(numEntries, analyzeSubscriptionBacklogResult.getFilterAcceptedEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRejectedEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledEntries()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledEntries()); + assertEquals(analyzeSubscriptionBacklogResult.getEntries(), numEntries); + assertEquals(analyzeSubscriptionBacklogResult.getFilterAcceptedEntries(), numEntries); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRejectedEntries(), 0); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledEntries(), 0); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledEntries(), 0); - assertEquals(numMessages, analyzeSubscriptionBacklogResult.getMessages()); - assertEquals(numMessages, analyzeSubscriptionBacklogResult.getFilterAcceptedMessages()); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRejectedMessages()); + assertEquals(analyzeSubscriptionBacklogResult.getMessages(), numMessages); + assertEquals(analyzeSubscriptionBacklogResult.getFilterAcceptedMessages(), numMessages); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRejectedMessages(), 0); - assertEquals(0, analyzeSubscriptionBacklogResult.getFilterRescheduledMessages()); + assertEquals(analyzeSubscriptionBacklogResult.getFilterRescheduledMessages(), 0); assertFalse(analyzeSubscriptionBacklogResult.isAborted()); } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java index 12ce7eb74c72b..b801d5f2b05a1 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/BitSetRecyclable.java @@ -216,6 +216,14 @@ public static BitSetRecyclable valueOf(byte[] bytes) { return BitSetRecyclable.valueOf(ByteBuffer.wrap(bytes)); } + /** + * Copy a BitSetRecyclable. + */ + public static BitSetRecyclable valueOf(BitSetRecyclable src) { + // The internal implementation will do the array-copy. + return valueOf(src.words); + } + /** * Returns a new bit set containing all the bits in the given byte * buffer between its position and limit. From dade5e0b41020b97d9b3d068055bf5d0b0f98324 Mon Sep 17 00:00:00 2001 From: thetumbled <52550727+thetumbled@users.noreply.github.com> Date: Thu, 22 Feb 2024 12:07:43 +0800 Subject: [PATCH 464/494] [fix] [txn] Get previous position by managed ledger. (#22024) --- .../buffer/impl/TopicTransactionBuffer.java | 4 +- .../buffer/TopicTransactionBufferTest.java | 200 +++++++++++++++++- 2 files changed, 197 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 89a8e95afba1f..7188ffb20d740 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -282,8 +282,8 @@ private void handleTransactionMessage(TxnID txnId, Position position) { .checkAbortedTransaction(txnId)) { ongoingTxns.put(txnId, (PositionImpl) position); PositionImpl firstPosition = ongoingTxns.get(ongoingTxns.firstKey()); - //max read position is less than first ongoing transaction message position, so entryId -1 - maxReadPosition = PositionImpl.get(firstPosition.getLedgerId(), firstPosition.getEntryId() - 1); + // max read position is less than first ongoing transaction message position + maxReadPosition = ((ManagedLedgerImpl) topic.getManagedLedger()).getPreviousPosition(firstPosition); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java index aa98fc7d70106..30cb3e765cba2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TopicTransactionBufferTest.java @@ -18,9 +18,11 @@ */ package org.apache.pulsar.broker.transaction.buffer; +import lombok.Cleanup; import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.service.BrokerService; @@ -31,7 +33,14 @@ import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBuffer; import org.apache.pulsar.broker.transaction.buffer.impl.TopicTransactionBufferState; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.TopicMessageId; import org.apache.pulsar.client.api.transaction.Transaction; +import org.apache.pulsar.client.impl.MessageIdImpl; +import org.apache.pulsar.client.impl.TopicMessageIdImpl; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID; import org.apache.pulsar.transaction.coordinator.TransactionMetadataStore; @@ -46,6 +55,7 @@ import java.time.Duration; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -53,6 +63,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.fail; + public class TopicTransactionBufferTest extends TransactionTestBase { @@ -130,7 +147,7 @@ public void testCheckDeduplicationFailedWhenCreatePersistentTopic() throws Excep } }) .when(brokerService) - .newTopic(Mockito.eq(topic), Mockito.any(), Mockito.eq(brokerService), + .newTopic(Mockito.eq(topic), any(), Mockito.eq(brokerService), Mockito.eq(PersistentTopic.class)); brokerService.createPersistentTopic0(topic, true, new CompletableFuture<>(), Collections.emptyMap()); @@ -138,7 +155,7 @@ public void testCheckDeduplicationFailedWhenCreatePersistentTopic() throws Excep Awaitility.waitAtMost(1, TimeUnit.MINUTES).until(() -> reference.get() != null); PersistentTopic persistentTopic = reference.get(); TransactionBuffer buffer = persistentTopic.getTransactionBuffer(); - Assert.assertTrue(buffer instanceof TopicTransactionBuffer); + assertTrue(buffer instanceof TopicTransactionBuffer); TopicTransactionBuffer ttb = (TopicTransactionBuffer) buffer; TopicTransactionBufferState.State expectState = TopicTransactionBufferState.State.Close; Assert.assertEquals(ttb.getState(), expectState); @@ -163,7 +180,7 @@ public void testCloseTransactionBufferWhenTimeout() throws Exception { return persistentTopic; }) .when(brokerService) - .newTopic(Mockito.eq(topic), Mockito.any(), Mockito.eq(brokerService), + .newTopic(Mockito.eq(topic), any(), Mockito.eq(brokerService), Mockito.eq(PersistentTopic.class)); CompletableFuture> f = brokerService.getTopic(topic, true); @@ -172,11 +189,184 @@ public void testCloseTransactionBufferWhenTimeout() throws Exception { .pollInterval(Duration.ofSeconds(2)).until(() -> reference.get() != null); PersistentTopic persistentTopic = reference.get(); TransactionBuffer buffer = persistentTopic.getTransactionBuffer(); - Assert.assertTrue(buffer instanceof TopicTransactionBuffer); + assertTrue(buffer instanceof TopicTransactionBuffer); TopicTransactionBuffer ttb = (TopicTransactionBuffer) buffer; TopicTransactionBufferState.State expectState = TopicTransactionBufferState.State.Close; Assert.assertEquals(ttb.getState(), expectState); - Assert.assertTrue(f.isCompletedExceptionally()); + assertTrue(f.isCompletedExceptionally()); + } + + /** + * This test mainly test the following two point: + * 1. `getLastMessageIds` will get max read position. + * Send two message |1:0|1:1|; mock max read position as |1:0|; `getLastMessageIds` will get |1:0|. + * 2. `getLastMessageIds` will wait Transaction buffer recover completely. + * Mock `checkIfTBRecoverCompletely` return an exception, `getLastMessageIds` will fail too. + * Mock `checkIfTBRecoverCompletely` return null, `getLastMessageIds` will get correct result. + */ + @Test + public void testGetMaxPositionAfterTBReady() throws Exception { + // 1. Prepare test environment. + String topic = "persistent://" + NAMESPACE1 + "/testGetMaxReadyPositionAfterTBReady"; + // 1.1 Mock component. + TransactionBuffer transactionBuffer = Mockito.spy(TransactionBuffer.class); + when(transactionBuffer.checkIfTBRecoverCompletely(anyBoolean())) + // Handle producer will check transaction buffer recover completely. + .thenReturn(CompletableFuture.completedFuture(null)) + // If the Transaction buffer failed to recover, we can not get the correct last max read id. + .thenReturn(CompletableFuture.failedFuture(new Throwable("Mock fail"))) + // If the transaction buffer recover successfully, the max read position can be acquired successfully. + .thenReturn(CompletableFuture.completedFuture(null)); + TransactionBufferProvider transactionBufferProvider = Mockito.spy(TransactionBufferProvider.class); + Mockito.doReturn(transactionBuffer).when(transactionBufferProvider).newTransactionBuffer(any()); + TransactionBufferProvider originalTBProvider = getPulsarServiceList().get(0).getTransactionBufferProvider(); + Mockito.doReturn(transactionBufferProvider).when(getPulsarServiceList().get(0)).getTransactionBufferProvider(); + // 2. Building producer and consumer. + admin.topics().createNonPartitionedTopic(topic); + @Cleanup + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("sub") + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + // 3. Send message and test the exception can be handled as expected. + MessageIdImpl messageId = (MessageIdImpl) producer.newMessage().send(); + producer.newMessage().send(); + Mockito.doReturn(new PositionImpl(messageId.getLedgerId(), messageId.getEntryId())) + .when(transactionBuffer).getMaxReadPosition(); + try { + consumer.getLastMessageIds(); + fail(); + } catch (PulsarClientException exception) { + assertTrue(exception.getMessage().contains("Failed to recover Transaction Buffer.")); + } + List messageIdList = consumer.getLastMessageIds(); + assertEquals(messageIdList.size(), 1); + TopicMessageIdImpl actualMessageID = (TopicMessageIdImpl) messageIdList.get(0); + assertEquals(messageId.getLedgerId(), actualMessageID.getLedgerId()); + assertEquals(messageId.getEntryId(), actualMessageID.getEntryId()); + // 4. Clean resource + Mockito.doReturn(originalTBProvider).when(getPulsarServiceList().get(0)).getTransactionBufferProvider(); + } + + /** + * Add a E2E test for the get last message ID. It tests 4 cases. + *

    + * 1. Only normal messages in the topic. + * 2. There are ongoing transactions, last message ID will not be updated until transaction end. + * 3. Aborted transaction will make the last message ID be updated as expected. + * 4. Committed transaction will make the last message ID be updated as expected. + *

    + */ + @Test + public void testGetLastMessageIdsWithOngoingTransactions() throws Exception { + // 1. Prepare environment + String topic = "persistent://" + NAMESPACE1 + "/testGetLastMessageIdsWithOngoingTransactions"; + String subName = "my-subscription"; + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscribe(); + + // 2. Test last max read position can be required correctly. + // 2.1 Case1: send 3 original messages. |1:0|1:1|1:2| + MessageIdImpl expectedLastMessageID = null; + for (int i = 0; i < 3; i++) { + expectedLastMessageID = (MessageIdImpl) producer.newMessage().send(); + } + assertMessageId(consumer, expectedLastMessageID, 0); + // 2.2 Case2: send 2 ongoing transactional messages and 2 original messages. + // |1:0|1:1|1:2|txn1->1:3|1:4|txn2->1:5|1:6|. + Transaction txn1 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + Transaction txn2 = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + producer.newMessage(txn1).send(); + MessageIdImpl expectedLastMessageID1 = (MessageIdImpl) producer.newMessage().send(); + producer.newMessage(txn2).send(); + MessageIdImpl expectedLastMessageID2 = (MessageIdImpl) producer.newMessage().send(); + // 2.2.1 Last message ID will not change when txn1 and txn2 do not end. + assertMessageId(consumer, expectedLastMessageID, 0); + // 2.2.2 Last message ID will update to 1:4 when txn1 committed. + txn1.commit().get(5, TimeUnit.SECONDS); + assertMessageId(consumer, expectedLastMessageID1, 0); + // 2.2.3 Last message ID will update to 1:6 when txn2 aborted. + txn2.abort().get(5, TimeUnit.SECONDS); + // Todo: We can not ignore the marker's position in this fix. + assertMessageId(consumer, expectedLastMessageID2, 2); + } + + /** + * produce 3 messages and then trigger a ledger switch, + * then create a transaction and send a transactional message. + * As there are messages in the new ledger, the reader should be able to read the messages. + * But reader.hasMessageAvailable() returns false if the entry id of max read position is -1. + * @throws Exception + */ + @Test + public void testGetLastMessageIdsWithOpenTransactionAtLedgerHead() throws Exception { + String topic = "persistent://" + NAMESPACE1 + "/testGetLastMessageIdsWithOpenTransactionAtLedgerHead"; + String subName = "my-subscription"; + @Cleanup + Producer producer = pulsarClient.newProducer() + .topic(topic) + .create(); + Consumer consumer = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName(subName) + .subscribe(); + MessageId expectedLastMessageID = null; + for (int i = 0; i < 3; i++) { + expectedLastMessageID = producer.newMessage().value(String.valueOf(i).getBytes()).send(); + System.out.println("expectedLastMessageID: " + expectedLastMessageID); + } + triggerLedgerSwitch(topic); + Transaction txn = pulsarClient.newTransaction() + .withTransactionTimeout(5, TimeUnit.HOURS) + .build() + .get(); + producer.newMessage(txn).send(); + + Reader reader = pulsarClient.newReader() + .topic(topic) + .startMessageId(MessageId.earliest) + .create(); + assertTrue(reader.hasMessageAvailable()); + } + + private void triggerLedgerSwitch(String topicName) throws Exception{ + admin.topics().unload(topicName); + Awaitility.await().until(() -> { + CompletableFuture> topicFuture = + getPulsarServiceList().get(0).getBrokerService().getTopic(topicName, false); + if (!topicFuture.isDone() || topicFuture.isCompletedExceptionally()){ + return false; + } + Optional topicOptional = topicFuture.join(); + if (!topicOptional.isPresent()){ + return false; + } + PersistentTopic persistentTopic = (PersistentTopic) topicOptional.get(); + ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger(); + return managedLedger.getState() == ManagedLedgerImpl.State.LedgerOpened; + }); + } + + private void assertMessageId(Consumer consumer, MessageIdImpl expected, int entryOffset) throws Exception { + TopicMessageIdImpl actual = (TopicMessageIdImpl) consumer.getLastMessageIds().get(0); + assertEquals(expected.getEntryId(), actual.getEntryId() - entryOffset); + assertEquals(expected.getLedgerId(), actual.getLedgerId()); } } From 55788734a53acd505d97e6f8b3b4bd567603d9ce Mon Sep 17 00:00:00 2001 From: Jiwe Guo Date: Fri, 23 Feb 2024 09:25:13 +0800 Subject: [PATCH 465/494] Fix compile issue --- .../broker/transaction/buffer/impl/TopicTransactionBuffer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 7188ffb20d740..8a76362f5a35f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -37,6 +37,7 @@ import org.apache.bookkeeper.mledger.ManagedCursor; import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl; import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.commons.collections4.map.LinkedMap; import org.apache.pulsar.broker.service.BrokerServiceException; From 90f76a7a3253f5c2dd46cd4fc44230e88c9fcb07 Mon Sep 17 00:00:00 2001 From: Frank Kelly <62910985+frankjkelly@users.noreply.github.com> Date: Tue, 19 Sep 2023 23:22:21 -0400 Subject: [PATCH 466/494] [improve][proxy] When adding new brokers resolve the DNS name more quickly (#21207) (cherry picked from commit e5b0f170e4295a22148ef686bd4ddf8143a222d0) --- docker/pulsar/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index b465bbf251da8..a4278ad1a717a 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -79,6 +79,7 @@ RUN mkdir -p /etc/apt/keyrings \ && apt-get -y install temurin-17-jdk \ && export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') \ && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-17-jdk-$ARCH/conf/security/java.security \ + && echo networkaddress.cache.negative.ttl=1 >> /usr/lib/jvm/temurin-17-jdk-$ARCH/conf/security/java.security \ # Cleanup apt RUN apt-get -y --purge autoremove \ From bbf6ddf9244c1dc02cc157a014c153a09f45be1f Mon Sep 17 00:00:00 2001 From: zifengmo <38554710+zifengmo@users.noreply.github.com> Date: Mon, 26 Feb 2024 18:06:28 +0800 Subject: [PATCH 467/494] [fix] [client] Do no retrying for error subscription not found when disabled allowAutoSubscriptionCreation (#22078) --- .../service/BrokerServiceException.java | 2 ++ .../client/api/MultiTopicsConsumerTest.java | 32 +++++++++++++++++++ .../client/api/PulsarClientException.java | 17 ++++++++++ .../apache/pulsar/client/impl/ClientCnx.java | 2 ++ 4 files changed, 53 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java index 3e77588b2459f..831d6068e2097 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerServiceException.java @@ -258,6 +258,8 @@ private static ServerError getClientErrorCode(Throwable t, boolean checkCauseIfU return ServerError.ServiceNotReady; } else if (t instanceof TopicNotFoundException) { return ServerError.TopicNotFound; + } else if (t instanceof SubscriptionNotFoundException) { + return ServerError.SubscriptionNotFound; } else if (t instanceof IncompatibleSchemaException || t instanceof InvalidSchemaDataException) { // for backward compatible with old clients, invalid schema data diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java index 315ce378d6953..bb8bab29ad9ef 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MultiTopicsConsumerTest.java @@ -371,4 +371,36 @@ public void testMultipleIOThreads() throws PulsarAdminException, PulsarClientExc assertTrue(consumer instanceof MultiTopicsConsumerImpl); assertTrue(consumer.isConnected()); } + + @Test(timeOut = 30000) + public void testSubscriptionNotFound() throws PulsarAdminException, PulsarClientException { + final var topic1 = newTopicName(); + final var topic2 = newTopicName(); + + pulsar.getConfiguration().setAllowAutoSubscriptionCreation(false); + + try { + final var singleTopicConsumer = pulsarClient.newConsumer() + .topic(topic1) + .subscriptionName("sub-1") + .isAckReceiptEnabled(true) + .subscribe(); + assertTrue(singleTopicConsumer instanceof ConsumerImpl); + } catch (Throwable t) { + assertTrue(t.getCause().getCause() instanceof PulsarClientException.SubscriptionNotFoundException); + } + + try { + final var multiTopicsConsumer = pulsarClient.newConsumer() + .topics(List.of(topic1, topic2)) + .subscriptionName("sub-2") + .isAckReceiptEnabled(true) + .subscribe(); + assertTrue(multiTopicsConsumer instanceof MultiTopicsConsumerImpl); + } catch (Throwable t) { + assertTrue(t.getCause().getCause() instanceof PulsarClientException.SubscriptionNotFoundException); + } + + pulsar.getConfiguration().setAllowAutoSubscriptionCreation(true); + } } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java index 9409eefe2e0f0..22a97571e532e 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/PulsarClientException.java @@ -344,6 +344,22 @@ public TopicDoesNotExistException(String msg) { } } + /** + * Not found subscription that cannot be created. + */ + public static class SubscriptionNotFoundException extends PulsarClientException { + /** + * Constructs an {@code SubscriptionNotFoundException} with the specified detail message. + * + * @param msg + * The detail message (which is saved for later retrieval + * by the {@link #getMessage()} method) + */ + public SubscriptionNotFoundException(String msg) { + super(msg); + } + } + /** * Lookup exception thrown by Pulsar client. */ @@ -1159,6 +1175,7 @@ public static boolean isRetriableError(Throwable t) { || t instanceof NotFoundException || t instanceof IncompatibleSchemaException || t instanceof TopicDoesNotExistException + || t instanceof SubscriptionNotFoundException || t instanceof UnsupportedAuthenticationException || t instanceof InvalidMessageException || t instanceof InvalidTopicNameException diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index f3e8b2354b344..496a9ca04271a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -1292,6 +1292,8 @@ public static PulsarClientException getPulsarClientException(ServerError error, return new PulsarClientException.IncompatibleSchemaException(errorMsg); case TopicNotFound: return new PulsarClientException.TopicDoesNotExistException(errorMsg); + case SubscriptionNotFound: + return new PulsarClientException.SubscriptionNotFoundException(errorMsg); case ConsumerAssignError: return new PulsarClientException.ConsumerAssignException(errorMsg); case NotAllowedError: From e616217c10dcf5959a856d5643613d9d0f6a57ef Mon Sep 17 00:00:00 2001 From: labuladong Date: Tue, 13 Jun 2023 16:40:54 +0800 Subject: [PATCH 468/494] [fix][broker] fix `Update contains no change` error when use `--update-auth-data` flag to update function/sink/source (#19450) Co-authored-by: tison (cherry picked from commit 2b92ed11051a1c51170bedab07be6acc55cd95c9) --- .../functions/utils/FunctionConfigUtils.java | 4 +- .../functions/utils/SinkConfigUtils.java | 2 +- .../worker/rest/api/FunctionsImpl.java | 3 +- .../functions/worker/rest/api/SinksImpl.java | 3 +- .../worker/rest/api/SourcesImpl.java | 3 +- .../api/v3/FunctionApiV3ResourceTest.java | 93 +++++++++++ .../rest/api/v3/SinkApiV3ResourceTest.java | 151 +++++++++++++----- .../rest/api/v3/SourceApiV3ResourceTest.java | 111 ++++++++++--- 8 files changed, 306 insertions(+), 64 deletions(-) diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index 15119050c57be..0a0f732e948a5 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -754,7 +754,7 @@ private static void verifyNoTopicClash(Collection inputTopics, String ou } } - private static void doCommonChecks(FunctionConfig functionConfig) { + public static void doCommonChecks(FunctionConfig functionConfig) { if (isEmpty(functionConfig.getTenant())) { throw new IllegalArgumentException("Function tenant cannot be null"); } @@ -896,7 +896,7 @@ private static void doCommonChecks(FunctionConfig functionConfig) { } } - private static Collection collectAllInputTopics(FunctionConfig functionConfig) { + public static Collection collectAllInputTopics(FunctionConfig functionConfig) { List retval = new LinkedList<>(); if (functionConfig.getInputs() != null) { retval.addAll(functionConfig.getInputs()); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index 7f974bb4b1945..1cf07173ec6aa 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -538,7 +538,7 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf return new ExtractedSinkDetails(sinkClassName, typeArg.getName(), functionClassName); } - private static Collection collectAllInputTopics(SinkConfig sinkConfig) { + public static Collection collectAllInputTopics(SinkConfig sinkConfig) { List retval = new LinkedList<>(); if (sinkConfig.getInputs() != null) { retval.addAll(sinkConfig.getInputs()); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 715d660ddff0d..078c47524f8e9 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -290,7 +290,8 @@ public void updateFunction(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, e.getMessage()); } - if (existingFunctionConfig.equals(mergedConfig) && isBlank(functionPkgUrl) && uploadedInputStream == null) { + if (existingFunctionConfig.equals(mergedConfig) && isBlank(functionPkgUrl) && uploadedInputStream == null + && (updateOptions == null || !updateOptions.isUpdateAuthData())) { log.error("{}/{}/{} Update contains no changes", tenant, namespace, functionName); throw new RestException(Response.Status.BAD_REQUEST, "Update contains no change"); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 5370fe93a7a30..c382ec9e01b35 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -295,7 +295,8 @@ public void updateSink(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, e.getMessage()); } - if (existingSinkConfig.equals(mergedConfig) && isBlank(sinkPkgUrl) && uploadedInputStream == null) { + if (existingSinkConfig.equals(mergedConfig) && isBlank(sinkPkgUrl) && uploadedInputStream == null + && (updateOptions == null || !updateOptions.isUpdateAuthData())) { log.error("{}/{}/{} Update contains no changes", tenant, namespace, sinkName); throw new RestException(Response.Status.BAD_REQUEST, "Update contains no change"); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index 2f491424d658a..c55ddf48b06b9 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -289,7 +289,8 @@ public void updateSource(final String tenant, throw new RestException(Response.Status.BAD_REQUEST, e.getMessage()); } - if (existingSourceConfig.equals(mergedConfig) && isBlank(sourcePkgUrl) && uploadedInputStream == null) { + if (existingSourceConfig.equals(mergedConfig) && isBlank(sourcePkgUrl) && uploadedInputStream == null + && (updateOptions == null || !updateOptions.isUpdateAuthData())) { log.error("{}/{}/{} Update contains no changes", tenant, namespace, sourceName); throw new RestException(Response.Status.BAD_REQUEST, "Update contains no change"); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index 337999bf2ad5e..0c20083bb89ca 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -28,6 +28,8 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Lists; import java.io.File; import java.io.FileInputStream; @@ -57,6 +59,7 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.FutureUtil; @@ -604,6 +607,96 @@ public void testUpdateMissingFunctionConfig() { null, null); } + @Test + public void testUpdateSourceWithNoChange() throws ClassNotFoundException { + mockWorkerUtils(); + + FunctionDetails functionDetails = createDefaultFunctionDetails(); + NarClassLoader mockedClassLoader = mock(NarClassLoader.class); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class}); + ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(),any(),any(),any())).thenCallRealMethod(); + ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); + }); + + doReturn(Function.class).when(mockedClassLoader).loadClass(anyString()); + + FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); + FunctionArchive functionArchive = FunctionArchive.builder() + .classLoader(mockedClassLoader) + .build(); + when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive); + when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath()); + + when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + // No change on config, + FunctionConfig funcConfig = createDefaultFunctionConfig(); + mockStatic(FunctionConfigUtils.class, ctx -> { + ctx.when(() -> FunctionConfigUtils.convertFromDetails(any())).thenReturn(funcConfig); + ctx.when(() -> FunctionConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionConfigUtils.convert(any(FunctionConfig.class), any(ClassLoader.class))).thenReturn(functionDetails); + ctx.when(() -> FunctionConfigUtils.convert(any(FunctionConfig.class), any(FunctionConfigUtils.ExtractedFunctionDetails.class))).thenReturn(functionDetails); + ctx.when(() -> FunctionConfigUtils.validateJavaFunction(any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionConfigUtils.doCommonChecks(any())).thenCallRealMethod(); + ctx.when(() -> FunctionConfigUtils.collectAllInputTopics(any())).thenCallRealMethod(); + ctx.when(() -> FunctionConfigUtils.doJavaChecks(any(), any())).thenCallRealMethod(); + }); + + // config has not changes and don't update auth, should fail + try { + resource.updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + null); + fail("Update without changes should fail"); + } catch (RestException e) { + assertTrue(e.getMessage().contains("Update contains no change")); + } + + try { + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(false); + resource.updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + updateOptions); + fail("Update without changes should fail"); + } catch (RestException e) { + assertTrue(e.getMessage().contains("Update contains no change")); + } + + // no changes but set the auth-update flag to true, should not fail + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(true); + resource.updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + updateOptions); + } + + private void registerDefaultFunction() { registerDefaultFunctionWithPackageUrl(null); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 9f786c5b73aac..5dcc795304ef5 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -18,21 +18,6 @@ */ package org.apache.pulsar.functions.worker.rest.api.v3; -import static org.apache.pulsar.functions.proto.Function.ProcessingGuarantees.ATLEAST_ONCE; -import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; import com.google.common.collect.Lists; import java.io.File; import java.io.FileInputStream; @@ -60,6 +45,7 @@ import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.nar.NarClassLoader; @@ -97,6 +83,23 @@ import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import static org.apache.pulsar.functions.proto.Function.ProcessingGuarantees.ATLEAST_ONCE; +import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; /** * Unit test of {@link SinksApiV3Resource}. @@ -965,29 +968,7 @@ private void testUpdateSinkMissingArguments( String className, Integer parallelism, String expectedError) throws Exception { - mockStatic(ConnectorUtils.class, ctx -> { - ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) - .thenReturn(CASSANDRA_STRING_SINK); - }); - - mockStatic(ClassLoaderUtils.class, ctx -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); - ctx.when(() -> FunctionCommon - .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) - .thenReturn(ATLEAST_ONCE); - }); - - this.mockedFunctionMetaData = - FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + mockFunctionCommon(tenant, namespace, sink); SinkConfig sinkConfig = new SinkConfig(); if (tenant != null) { @@ -1026,6 +1007,32 @@ private void testUpdateSinkMissingArguments( } + private void mockFunctionCommon(String tenant, String namespace, String sink) throws IOException { + mockStatic(ConnectorUtils.class, ctx -> { + ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) + .thenReturn(CASSANDRA_STRING_SINK); + }); + + mockStatic(ClassLoaderUtils.class, ctx -> { + }); + + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); + ctx.when(() -> FunctionCommon + .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) + .thenReturn(ATLEAST_ONCE); + }); + + this.mockedFunctionMetaData = + FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + } + private void updateDefaultSink() throws Exception { updateDefaultSinkWithPackageUrl(null); } @@ -1848,4 +1855,72 @@ public void testRegisterSinkSuccessK8sWithUpload() throws Exception { } } } + + @Test + public void testUpdateSinkWithNoChange() throws IOException { + mockWorkerUtils(); + + // No change on config, + SinkConfig sinkConfig = createDefaultSinkConfig(); + + mockStatic(SinkConfigUtils.class, ctx -> { + ctx.when(() -> SinkConfigUtils.convertFromDetails(any())).thenReturn(sinkConfig); + ctx.when(() -> SinkConfigUtils.convert(any(), any())).thenCallRealMethod(); + ctx.when(() -> SinkConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); + ctx.when(() -> SinkConfigUtils.clone(any())).thenCallRealMethod(); + ctx.when(() -> SinkConfigUtils.collectAllInputTopics(any())).thenCallRealMethod(); + ctx.when(() -> SinkConfigUtils.validateAndExtractDetails(any(),any(),any(),anyBoolean())).thenCallRealMethod(); + }); + + mockFunctionCommon(sinkConfig.getTenant(), sinkConfig.getNamespace(), sinkConfig.getName()); + + // config has not changes and don't update auth, should fail + try { + resource.updateSink( + sinkConfig.getTenant(), + sinkConfig.getNamespace(), + sinkConfig.getName(), + null, + mockedFormData, + null, + sinkConfig, + null, + null); + fail("Update without changes should fail"); + } catch (RestException e) { + assertTrue(e.getMessage().contains("Update contains no change")); + } + + try { + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(false); + resource.updateSink( + sinkConfig.getTenant(), + sinkConfig.getNamespace(), + sinkConfig.getName(), + null, + mockedFormData, + null, + sinkConfig, + null, + updateOptions); + fail("Update without changes should fail"); + } catch (RestException e) { + assertTrue(e.getMessage().contains("Update contains no change")); + } + + // no changes but set the auth-update flag to true, should not fail + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(true); + resource.updateSink( + sinkConfig.getTenant(), + sinkConfig.getNamespace(), + sinkConfig.getName(), + null, + mockedFormData, + null, + sinkConfig, + null, + updateOptions); + } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java index 36be2de716661..eabf954cc77b1 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java @@ -22,6 +22,7 @@ import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOInvalidNar; import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOTwitterNar; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; @@ -33,6 +34,7 @@ import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Lists; import java.io.File; @@ -58,6 +60,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.admin.Tenants; +import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.nar.NarClassLoader; @@ -846,6 +849,72 @@ public void testUpdateSourceChangedTopic() throws Exception { null); } + @Test + public void testUpdateSourceWithNoChange() { + mockWorkerUtils(); + + // No change on config, + SourceConfig sourceConfig = createDefaultSourceConfig(); + mockStatic(SourceConfigUtils.class, ctx -> { + ctx.when(() -> SourceConfigUtils.convertFromDetails(any())).thenReturn(sourceConfig); + ctx.when(() -> SourceConfigUtils.convert(any(), any())).thenCallRealMethod(); + ctx.when(() -> SourceConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); + ctx.when(() -> SourceConfigUtils.clone(any())).thenCallRealMethod(); + ctx.when(() -> SourceConfigUtils.validateAndExtractDetails(any(),any(),anyBoolean())).thenCallRealMethod(); + }); + + mockFunctionCommon(sourceConfig.getTenant(), sourceConfig.getNamespace(), sourceConfig.getName()); + + // config has not changes and don't update auth, should fail + try { + resource.updateSource( + sourceConfig.getTenant(), + sourceConfig.getNamespace(), + sourceConfig.getName(), + null, + mockedFormData, + null, + sourceConfig, + null, + null); + fail("Update without changes should fail"); + } catch (RestException e) { + assertTrue(e.getMessage().contains("Update contains no change")); + } + + try { + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(false); + resource.updateSource( + sourceConfig.getTenant(), + sourceConfig.getNamespace(), + sourceConfig.getName(), + null, + mockedFormData, + null, + sourceConfig, + null, + updateOptions); + fail("Update without changes should fail"); + } catch (RestException e) { + assertTrue(e.getMessage().contains("Update contains no change")); + } + + // no changes but set the auth-update flag to true, should not fail + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(true); + resource.updateSource( + sourceConfig.getTenant(), + sourceConfig.getNamespace(), + sourceConfig.getName(), + null, + mockedFormData, + null, + sourceConfig, + null, + updateOptions); + } + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source parallelism must be a " + "positive number") public void testUpdateSourceZeroParallelism() throws Exception { @@ -881,26 +950,7 @@ private void testUpdateSourceMissingArguments( Integer parallelism, String expectedError) throws Exception { - mockStatic(ConnectorUtils.class, c -> { - }); - mockStatic(ClassLoaderUtils.class, c -> { - }); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) - .thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) - .thenReturn(narClassLoader); - - - }); - - this.mockedFunctionMetaData = - FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + mockFunctionCommon(tenant, namespace, function); SourceConfig sourceConfig = new SourceConfig(); if (tenant != null) { @@ -942,6 +992,27 @@ private void testUpdateSourceMissingArguments( } + private void mockFunctionCommon(String tenant, String namespace, String function) { + mockStatic(ConnectorUtils.class, c -> { + }); + mockStatic(ClassLoaderUtils.class, c -> { + }); + mockStatic(FunctionCommon.class, ctx -> { + ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); + ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) + .thenReturn(String.class); + ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) + .thenReturn(narClassLoader); + }); + + this.mockedFunctionMetaData = + FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + } + private void updateDefaultSource() throws Exception { updateDefaultSourceWithPackageUrl(null); } From 55db3f195c7e11d60b7db9109664c31cfbfe71b5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 26 Feb 2024 13:50:10 +0200 Subject: [PATCH 469/494] [improve][fn] Optimize Function Worker startup by lazy loading and direct zip/bytecode access (#22122) (cherry picked from commit bbc62245c5ddba1de4b1e7cee4ab49334bc36277) --- conf/functions_worker.yml | 15 +- .../server/src/assemble/LICENSE.bin.txt | 4 + pom.xml | 14 + .../pulsar/common/nar/NarClassLoader.java | 33 +- .../apache/pulsar/common/nar/NarUnpacker.java | 27 +- .../apache/pulsar/functions/LocalRunner.java | 45 +- .../runtime/JavaInstanceStarter.java | 19 +- .../runtime/thread/ThreadRuntime.java | 6 +- .../functions/worker/ConnectorsManager.java | 46 +- .../functions/worker/FunctionsManager.java | 44 +- .../pulsar/functions/worker/WorkerConfig.java | 21 +- pulsar-functions/utils/pom.xml | 11 + .../functions/utils/FunctionCommon.java | 311 +-- .../functions/utils/FunctionConfigUtils.java | 155 +- .../functions/utils/FunctionFilePackage.java | 179 ++ .../utils/FunctionRuntimeCommon.java | 170 ++ .../utils/LoadedFunctionPackage.java | 89 + .../functions/utils/SinkConfigUtils.java | 125 +- .../functions/utils/SourceConfigUtils.java | 82 +- .../utils/ValidatableFunctionPackage.java | 59 + .../functions/utils/ValidatorUtils.java | 207 +- .../utils/functions/FunctionArchive.java | 52 +- .../utils/functions/FunctionUtils.java | 74 +- .../pulsar/functions/utils/io/Connector.java | 76 +- .../functions/utils/io/ConnectorUtils.java | 153 +- .../functions/utils/FunctionCommonTest.java | 81 +- .../utils/FunctionConfigUtilsTest.java | 59 +- .../functions/utils/SinkConfigUtilsTest.java | 67 +- .../functions/worker/FunctionActioner.java | 13 +- .../functions/worker/PulsarWorkerService.java | 8 + .../worker/rest/api/ComponentImpl.java | 11 +- .../worker/rest/api/FunctionsImpl.java | 40 +- .../functions/worker/rest/api/SinksImpl.java | 59 +- .../worker/rest/api/SourcesImpl.java | 33 +- .../worker/rest/api/FunctionsImplTest.java | 2 +- .../api/v2/FunctionApiV2ResourceTest.java | 1446 +----------- .../v3/AbstractFunctionApiResourceTest.java | 1367 +++++++++++ .../api/v3/AbstractFunctionsResourceTest.java | 323 +++ .../api/v3/FunctionApiV3ResourceTest.java | 2079 +++-------------- .../rest/api/v3/SinkApiV3ResourceTest.java | 451 +--- .../rest/api/v3/SourceApiV3ResourceTest.java | 354 +-- .../conf/functions_worker.conf | 2 +- .../integration/topologies/PulsarCluster.java | 39 +- 43 files changed, 3642 insertions(+), 4809 deletions(-) create mode 100644 pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java create mode 100644 pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java create mode 100644 pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java create mode 100644 pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java create mode 100644 pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java create mode 100644 pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index bb15e0ca416da..05c875adc4c64 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -43,6 +43,16 @@ metadataStoreOperationTimeoutSeconds: 30 # Metadata store cache expiry time in seconds metadataStoreCacheExpirySeconds: 300 +# Specifies if the function worker should use classloading for validating submissions for built-in +# connectors and functions. This is required for validateConnectorConfig to take effect. +# Default is false. +enableClassloadingOfBuiltinFiles: false + +# Specifies if the function worker should use classloading for validating submissions for external +# connectors and functions. This is required for validateConnectorConfig to take effect. +# Default is false. +enableClassloadingOfExternalFiles: false + ################################ # Function package management ################################ @@ -400,7 +410,10 @@ saslJaasServerRoleTokenSignerSecretPath: connectorsDirectory: ./connectors functionsDirectory: ./functions -# Should connector config be validated during submission +# Enables extended validation for connector config with fine-grain annotation based validation +# during submission. Classloading with either enableClassloadingOfExternalFiles or +# enableClassloadingOfBuiltinFiles must be enabled on the worker for this to take effect. +# Default is false. validateConnectorConfig: false # Whether to initialize distributed log metadata by runtime. diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 77d7be44478db..87216c9e32250 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -445,6 +445,10 @@ The Apache Software License, Version 2.0 * Jodah - net.jodah-typetools-0.5.0.jar - net.jodah-failsafe-2.4.4.jar + * Byte Buddy + - net.bytebuddy-byte-buddy-1.14.12.jar + * zt-zip + - org.zeroturnaround-zt-zip-1.17.jar * Apache Avro - org.apache.avro-avro-1.11.3.jar - org.apache.avro-avro-protobuf-1.11.3.jar diff --git a/pom.xml b/pom.xml index 180a0ac59d311..6e4b31aa28957 100644 --- a/pom.xml +++ b/pom.xml @@ -162,6 +162,8 @@ flexible messaging model and an intuitive client API. 0.43.3 true 0.5.0 + 1.14.12 + 1.17 3.19.6 ${protobuf3.version} 1.55.3 @@ -1060,6 +1062,18 @@ flexible messaging model and an intuitive client API. ${typetools.version} + + net.bytebuddy + byte-buddy + ${byte-buddy.version} + + + + org.zeroturnaround + zt-zip + ${zt-zip.version} + + io.grpc grpc-bom diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java index 620e1156d3555..9736d8b47ef71 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarClassLoader.java @@ -154,6 +154,11 @@ public NarClassLoader run() { }); } + public static List getClasspathFromArchive(File narPath, String narExtractionDirectory) throws IOException { + File unpacked = NarUnpacker.unpackNar(narPath, getNarExtractionDirectory(narExtractionDirectory)); + return getClassPathEntries(unpacked); + } + private static File getNarExtractionDirectory(String configuredDirectory) { return new File(configuredDirectory + "/" + TMP_DIR_PREFIX); } @@ -164,16 +169,11 @@ private static File getNarExtractionDirectory(String configuredDirectory) { * @param narWorkingDirectory * directory to explode nar contents to * @param parent - * @throws IllegalArgumentException - * if the NAR is missing the Java Services API file for FlowFileProcessor implementations. - * @throws ClassNotFoundException - * if any of the FlowFileProcessor implementations defined by the Java Services API cannot be - * loaded. * @throws IOException * if an error occurs while loading the NAR. */ private NarClassLoader(final File narWorkingDirectory, Set additionalJars, ClassLoader parent) - throws ClassNotFoundException, IOException { + throws IOException { super(new URL[0], parent); this.narWorkingDirectory = narWorkingDirectory; @@ -238,22 +238,31 @@ public List getServiceImplementation(String serviceName) throws IOExcept * if the URL list could not be updated. */ private void updateClasspath(File root) throws IOException { - addURL(root.toURI().toURL()); // for compiled classes, META-INF/, etc. + getClassPathEntries(root).forEach(f -> { + try { + addURL(f.toURI().toURL()); + } catch (IOException e) { + log.error("Failed to add entry to classpath: {}", f, e); + } + }); + } + static List getClassPathEntries(File root) { + List classPathEntries = new ArrayList<>(); + classPathEntries.add(root); File dependencies = new File(root, "META-INF/bundled-dependencies"); if (!dependencies.isDirectory()) { - log.warn("{} does not contain META-INF/bundled-dependencies!", narWorkingDirectory); + log.warn("{} does not contain META-INF/bundled-dependencies!", root); } - addURL(dependencies.toURI().toURL()); + classPathEntries.add(dependencies); if (dependencies.isDirectory()) { final File[] jarFiles = dependencies.listFiles(JAR_FILTER); if (jarFiles != null) { Arrays.sort(jarFiles, Comparator.comparing(File::getName)); - for (File libJar : jarFiles) { - addURL(libJar.toURI().toURL()); - } + classPathEntries.addAll(Arrays.asList(jarFiles)); } } + return classPathEntries; } @Override diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java index 9bd5bc48df819..1e34c3e4fe706 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/nar/NarUnpacker.java @@ -32,13 +32,14 @@ import java.io.RandomAccessFile; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; +import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.Enumeration; import java.util.concurrent.ConcurrentHashMap; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import lombok.extern.slf4j.Slf4j; /** @@ -113,18 +114,24 @@ static File doUnpackNar(final File nar, final File baseWorkingDirectory, Runnabl * if the NAR could not be unpacked. */ private static void unpack(final File nar, final File workingDirectory) throws IOException { - try (JarFile jarFile = new JarFile(nar)) { - Enumeration jarEntries = jarFile.entries(); - while (jarEntries.hasMoreElements()) { - JarEntry jarEntry = jarEntries.nextElement(); - String name = jarEntry.getName(); - File f = new File(workingDirectory, name); - if (jarEntry.isDirectory()) { + Path workingDirectoryPath = workingDirectory.toPath().normalize(); + try (ZipFile zipFile = new ZipFile(nar)) { + Enumeration zipEntries = zipFile.entries(); + while (zipEntries.hasMoreElements()) { + ZipEntry zipEntry = zipEntries.nextElement(); + String name = zipEntry.getName(); + Path targetFilePath = workingDirectoryPath.resolve(name).normalize(); + if (!targetFilePath.startsWith(workingDirectoryPath)) { + log.error("Invalid zip file with entry '{}'", name); + throw new IOException("Invalid zip file. Aborting unpacking."); + } + File f = targetFilePath.toFile(); + if (zipEntry.isDirectory()) { FileUtils.ensureDirectoryExistAndCanReadAndWrite(f); } else { // The directory entry might appear after the file entry FileUtils.ensureDirectoryExistAndCanReadAndWrite(f.getParentFile()); - makeFile(jarFile.getInputStream(jarEntry), f); + makeFile(zipFile.getInputStream(zipEntry), f); } } } diff --git a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java index ed9b0af3b43d8..711fa33edb2a2 100644 --- a/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java +++ b/pulsar-functions/localrun/src/main/java/org/apache/pulsar/functions/LocalRunner.java @@ -52,7 +52,9 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.Utils; +import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.nar.FileUtils; @@ -75,8 +77,11 @@ import org.apache.pulsar.functions.secretsproviderconfigurator.SecretsProviderConfigurator; import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionRuntimeCommon; +import org.apache.pulsar.functions.utils.LoadedFunctionPackage; import org.apache.pulsar.functions.utils.SinkConfigUtils; import org.apache.pulsar.functions.utils.SourceConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.functions.FunctionUtils; @@ -357,9 +362,12 @@ public void start(boolean blocking) throws Exception { userCodeFile = functionConfig.getJar(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.FUNCTION, functionConfig.getClassName()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), + FunctionDefinition.class); functionDetails = FunctionConfigUtils.convert( functionConfig, - FunctionConfigUtils.validateJavaFunction(functionConfig, getCurrentOrUserCodeClassLoader())); + FunctionConfigUtils.validateJavaFunction(functionConfig, validatableFunctionPackage)); } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) { userCodeFile = functionConfig.getGo(); } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) { @@ -369,7 +377,10 @@ public void start(boolean blocking) throws Exception { } if (functionDetails == null) { - functionDetails = FunctionConfigUtils.convert(functionConfig, getCurrentOrUserCodeClassLoader()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), + FunctionDefinition.class); + functionDetails = FunctionConfigUtils.convert(functionConfig, validatableFunctionPackage); } } else if (sourceConfig != null) { inferMissingArguments(sourceConfig); @@ -377,9 +388,10 @@ public void start(boolean blocking) throws Exception { parallelism = sourceConfig.getParallelism(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.SOURCE, sourceConfig.getClassName()); - functionDetails = SourceConfigUtils.convert( - sourceConfig, - SourceConfigUtils.validateAndExtractDetails(sourceConfig, getCurrentOrUserCodeClassLoader(), true)); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), ConnectorDefinition.class); + functionDetails = SourceConfigUtils.convert(sourceConfig, + SourceConfigUtils.validateAndExtractDetails(sourceConfig, validatableFunctionPackage, true)); } else if (sinkConfig != null) { inferMissingArguments(sinkConfig); userCodeFile = sinkConfig.getArchive(); @@ -387,6 +399,8 @@ public void start(boolean blocking) throws Exception { parallelism = sinkConfig.getParallelism(); userCodeClassLoader = extractClassLoader( userCodeFile, ComponentType.SINK, sinkConfig.getClassName()); + ValidatableFunctionPackage validatableFunctionPackage = + new LoadedFunctionPackage(getCurrentOrUserCodeClassLoader(), ConnectorDefinition.class); if (isNotEmpty(sinkConfig.getTransformFunction())) { transformFunctionCodeClassLoader = extractClassLoader( sinkConfig.getTransformFunction(), @@ -395,16 +409,19 @@ public void start(boolean blocking) throws Exception { } ClassLoader functionClassLoader = null; + ValidatableFunctionPackage validatableTransformFunction = null; if (transformFunctionCodeClassLoader != null) { functionClassLoader = transformFunctionCodeClassLoader.getClassLoader() == null ? Thread.currentThread().getContextClassLoader() : transformFunctionCodeClassLoader.getClassLoader(); + validatableTransformFunction = + new LoadedFunctionPackage(functionClassLoader, FunctionDefinition.class); } functionDetails = SinkConfigUtils.convert( sinkConfig, - SinkConfigUtils.validateAndExtractDetails(sinkConfig, getCurrentOrUserCodeClassLoader(), - functionClassLoader, true)); + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunctionPackage, + validatableTransformFunction, true)); } else { throw new IllegalArgumentException("Must specify Function, Source or Sink config"); } @@ -472,7 +489,7 @@ private UserCodeClassLoader extractClassLoader(String userCodeFile, ComponentTyp if (classLoader == null) { if (userCodeFile != null && Utils.isFunctionPackageUrlSupported(userCodeFile)) { File file = FunctionCommon.extractFileFromPkgURL(userCodeFile); - classLoader = FunctionCommon.getClassLoaderFromPackage( + classLoader = FunctionRuntimeCommon.getClassLoaderFromPackage( componentType, className, file, narExtractionDirectory); classLoaderCreated = true; } else if (userCodeFile != null) { @@ -494,7 +511,7 @@ private UserCodeClassLoader extractClassLoader(String userCodeFile, ComponentTyp } throw new RuntimeException(errorMsg + " (" + userCodeFile + ") does not exist"); } - classLoader = FunctionCommon.getClassLoaderFromPackage( + classLoader = FunctionRuntimeCommon.getClassLoaderFromPackage( componentType, className, file, narExtractionDirectory); classLoaderCreated = true; } else { @@ -713,7 +730,7 @@ private ClassLoader isBuiltInFunction(String functionType) throws IOException { FunctionArchive function = functions.get(functionName); if (function != null && function.getFunctionDefinition().getFunctionClass() != null) { // Function type is a valid built-in type. - return function.getClassLoader(); + return function.getFunctionPackage().getClassLoader(); } else { return null; } @@ -727,7 +744,7 @@ private ClassLoader isBuiltInSource(String sourceType) throws IOException { Connector connector = connectors.get(source); if (connector != null && connector.getConnectorDefinition().getSourceClass() != null) { // Source type is a valid built-in connector type. - return connector.getClassLoader(); + return connector.getConnectorFunctionPackage().getClassLoader(); } else { return null; } @@ -741,18 +758,18 @@ private ClassLoader isBuiltInSink(String sinkType) throws IOException { Connector connector = connectors.get(sink); if (connector != null && connector.getConnectorDefinition().getSinkClass() != null) { // Sink type is a valid built-in connector type - return connector.getClassLoader(); + return connector.getConnectorFunctionPackage().getClassLoader(); } else { return null; } } private TreeMap getFunctions() throws IOException { - return FunctionUtils.searchForFunctions(functionsDir); + return FunctionUtils.searchForFunctions(functionsDir, narExtractionDirectory, true); } private TreeMap getConnectors() throws IOException { - return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory); + return ConnectorUtils.searchForConnectors(connectorsDir, narExtractionDirectory, true); } private SecretsProviderConfigurator getSecretsProviderConfigurator() { diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java index b78f07076f674..8436cbaf42e09 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceStarter.java @@ -38,6 +38,9 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.pool.TypePool; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.WindowConfig; import org.apache.pulsar.common.nar.NarClassLoader; @@ -318,7 +321,8 @@ public void close() { } private void inferringMissingTypeClassName(Function.FunctionDetails.Builder functionDetailsBuilder, - ClassLoader classLoader) throws ClassNotFoundException { + ClassLoader classLoader) { + TypePool typePool = TypePool.Default.of(ClassFileLocator.ForClassLoader.of(classLoader)); switch (functionDetailsBuilder.getComponentType()) { case FUNCTION: if ((functionDetailsBuilder.hasSource() @@ -337,14 +341,13 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func WindowConfig.class); className = windowConfig.getActualWindowFunctionClassName(); } - - Class[] typeArgs = FunctionCommon.getFunctionTypes(classLoader.loadClass(className), + TypeDefinition[] typeArgs = FunctionCommon.getFunctionTypes(typePool.describe(className).resolve(), isWindowConfigPresent); if (functionDetailsBuilder.hasSource() && functionDetailsBuilder.getSource().getTypeClassName().isEmpty() && typeArgs[0] != null) { Function.SourceSpec.Builder sourceBuilder = functionDetailsBuilder.getSource().toBuilder(); - sourceBuilder.setTypeClassName(typeArgs[0].getName()); + sourceBuilder.setTypeClassName(typeArgs[0].asErasure().getTypeName()); functionDetailsBuilder.setSource(sourceBuilder.build()); } @@ -352,7 +355,7 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func && functionDetailsBuilder.getSink().getTypeClassName().isEmpty() && typeArgs[1] != null) { Function.SinkSpec.Builder sinkBuilder = functionDetailsBuilder.getSink().toBuilder(); - sinkBuilder.setTypeClassName(typeArgs[1].getName()); + sinkBuilder.setTypeClassName(typeArgs[1].asErasure().getTypeName()); functionDetailsBuilder.setSink(sinkBuilder.build()); } } @@ -361,7 +364,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func if ((functionDetailsBuilder.hasSink() && functionDetailsBuilder.getSink().getTypeClassName().isEmpty())) { String typeArg = - getSinkType(functionDetailsBuilder.getSink().getClassName(), classLoader).getName(); + getSinkType(functionDetailsBuilder.getSink().getClassName(), typePool).asErasure() + .getTypeName(); Function.SinkSpec.Builder sinkBuilder = Function.SinkSpec.newBuilder(functionDetailsBuilder.getSink()); @@ -380,7 +384,8 @@ private void inferringMissingTypeClassName(Function.FunctionDetails.Builder func if ((functionDetailsBuilder.hasSource() && functionDetailsBuilder.getSource().getTypeClassName().isEmpty())) { String typeArg = - getSourceType(functionDetailsBuilder.getSource().getClassName(), classLoader).getName(); + getSourceType(functionDetailsBuilder.getSource().getClassName(), typePool).asErasure() + .getTypeName(); Function.SourceSpec.Builder sourceBuilder = Function.SourceSpec.newBuilder(functionDetailsBuilder.getSource()); diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java index ed128568bcf50..9dca4015d5ef5 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/thread/ThreadRuntime.java @@ -124,17 +124,17 @@ private static ClassLoader getFunctionClassLoader(InstanceConfig instanceConfig, if (componentType == Function.FunctionDetails.ComponentType.FUNCTION && functionsManager.isPresent()) { return functionsManager.get() .getFunction(instanceConfig.getFunctionDetails().getBuiltin()) - .getClassLoader(); + .getFunctionPackage().getClassLoader(); } if (componentType == Function.FunctionDetails.ComponentType.SOURCE && connectorsManager.isPresent()) { return connectorsManager.get() .getConnector(instanceConfig.getFunctionDetails().getSource().getBuiltin()) - .getClassLoader(); + .getConnectorFunctionPackage().getClassLoader(); } if (componentType == Function.FunctionDetails.ComponentType.SINK && connectorsManager.isPresent()) { return connectorsManager.get() .getConnector(instanceConfig.getFunctionDetails().getSink().getBuiltin()) - .getClassLoader(); + .getConnectorFunctionPackage().getClassLoader(); } } return loadJars(jarFile, instanceConfig, functionId, instanceConfig.getFunctionDetails().getName(), diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java index e1770b8b64415..19d31d0f63b1d 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/ConnectorsManager.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.worker; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.file.Path; import java.util.List; @@ -27,18 +28,35 @@ import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.utils.io.ConnectorUtils; @Slf4j -public class ConnectorsManager { +public class ConnectorsManager implements AutoCloseable { @Getter private volatile TreeMap connectors; + @VisibleForTesting + public ConnectorsManager() { + this.connectors = new TreeMap<>(); + } + public ConnectorsManager(WorkerConfig workerConfig) throws IOException { - this.connectors = ConnectorUtils - .searchForConnectors(workerConfig.getConnectorsDirectory(), workerConfig.getNarExtractionDirectory()); + this.connectors = createConnectors(workerConfig); + } + + private static TreeMap createConnectors(WorkerConfig workerConfig) throws IOException { + boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles() + || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + return ConnectorUtils.searchForConnectors(workerConfig.getConnectorsDirectory(), + workerConfig.getNarExtractionDirectory(), enableClassloading); + } + + @VisibleForTesting + public void addConnector(String connectorType, Connector connector) { + connectors.put(connectorType, connector); } public Connector getConnector(String connectorType) { @@ -71,7 +89,25 @@ public Path getSinkArchive(String sinkType) { } public void reloadConnectors(WorkerConfig workerConfig) throws IOException { - connectors = ConnectorUtils - .searchForConnectors(workerConfig.getConnectorsDirectory(), workerConfig.getNarExtractionDirectory()); + TreeMap oldConnectors = connectors; + this.connectors = createConnectors(workerConfig); + closeConnectors(oldConnectors); } + + @Override + public void close() { + closeConnectors(connectors); + } + + private void closeConnectors(TreeMap connectorMap) { + connectorMap.values().forEach(connector -> { + try { + connector.close(); + } catch (Exception e) { + log.warn("Failed to close connector", e); + } + }); + connectorMap.clear(); + } + } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java index 9199d568cad03..5ab7ff7221abb 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/FunctionsManager.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.worker; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.file.Path; import java.util.List; @@ -25,16 +26,25 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.functions.FunctionUtils; @Slf4j -public class FunctionsManager { - +public class FunctionsManager implements AutoCloseable { private TreeMap functions; + @VisibleForTesting + public FunctionsManager() { + this.functions = new TreeMap<>(); + } + public FunctionsManager(WorkerConfig workerConfig) throws IOException { - this.functions = FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory()); + this.functions = createFunctions(workerConfig); + } + + public void addFunction(String functionType, FunctionArchive functionArchive) { + functions.put(functionType, functionArchive); } public FunctionArchive getFunction(String functionType) { @@ -51,6 +61,32 @@ public List getFunctionDefinitions() { } public void reloadFunctions(WorkerConfig workerConfig) throws IOException { - this.functions = FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory()); + TreeMap oldFunctions = functions; + this.functions = createFunctions(workerConfig); + closeFunctions(oldFunctions); + } + + private static TreeMap createFunctions(WorkerConfig workerConfig) throws IOException { + boolean enableClassloading = workerConfig.getEnableClassloadingOfBuiltinFiles() + || ThreadRuntimeFactory.class.getName().equals(workerConfig.getFunctionRuntimeFactoryClassName()); + return FunctionUtils.searchForFunctions(workerConfig.getFunctionsDirectory(), + workerConfig.getNarExtractionDirectory(), + enableClassloading); + } + + @Override + public void close() { + closeFunctions(functions); + } + + private void closeFunctions(TreeMap functionMap) { + functionMap.values().forEach(functionArchive -> { + try { + functionArchive.close(); + } catch (Exception e) { + log.warn("Failed to close function archive", e); + } + }); + functionMap.clear(); } } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java index 3b8ddf774d162..8dfd91e0b20cf 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java @@ -238,6 +238,22 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { ) private boolean zooKeeperAllowReadOnlyOperations; + @FieldContext( + category = CATEGORY_WORKER, + doc = "Specifies if the function worker should use classloading for validating submissions for built-in " + + "connectors and functions. This is required for validateConnectorConfig to take effect. " + + "Default is false." + ) + private Boolean enableClassloadingOfBuiltinFiles = false; + + @FieldContext( + category = CATEGORY_WORKER, + doc = "Specifies if the function worker should use classloading for validating submissions for external " + + "connectors and functions. This is required for validateConnectorConfig to take effect. " + + "Default is false." + ) + private Boolean enableClassloadingOfExternalFiles = false; + @FieldContext( category = CATEGORY_CONNECTORS, doc = "The path to the location to locate builtin connectors" @@ -250,7 +266,10 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { private String narExtractionDirectory = NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR; @FieldContext( category = CATEGORY_CONNECTORS, - doc = "Should we validate connector config during submission" + doc = "Enables extended validation for connector config with fine-grain annotation based validation " + + "during submission. Classloading with either enableClassloadingOfExternalFiles or " + + "enableClassloadingOfBuiltinFiles must be enabled on the worker for this to take effect. " + + "Default is false." ) private Boolean validateConnectorConfig = false; @FieldContext( diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index dd3b394ab9501..3f836cae530c8 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -87,6 +87,17 @@ typetools + + net.bytebuddy + byte-buddy + + + + org.zeroturnaround + zt-zip + 1.17 + + ${project.groupId} pulsar-client-original diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java index 7df173da0f195..6a3d2f6ad7ddb 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionCommon.java @@ -22,16 +22,9 @@ import com.google.protobuf.AbstractMessage.Builder; import com.google.protobuf.MessageOrBuilder; import com.google.protobuf.util.JsonFormat; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.ObjectOutputStream; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.net.MalformedURLException; import java.net.ServerSocket; import java.net.URISyntaxException; import java.net.URL; @@ -41,10 +34,14 @@ import java.util.Collection; import java.util.Map; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.description.type.TypeList; +import net.bytebuddy.pool.TypePool; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.MessageId; @@ -54,16 +51,11 @@ import org.apache.pulsar.client.impl.auth.AuthenticationDataBasic; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.Utils; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarClassLoaderBuilder; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.WindowFunction; import org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType; import org.apache.pulsar.functions.proto.Function.FunctionDetails.Runtime; -import org.apache.pulsar.functions.utils.functions.FunctionUtils; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; import org.apache.pulsar.io.core.BatchSource; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.Source; @@ -97,50 +89,74 @@ public static int findAvailablePort() { } } - public static Class[] getFunctionTypes(FunctionConfig functionConfig, ClassLoader classLoader) + public static TypeDefinition[] getFunctionTypes(FunctionConfig functionConfig, TypePool typePool) throws ClassNotFoundException { - return getFunctionTypes(functionConfig, classLoader.loadClass(functionConfig.getClassName())); + return getFunctionTypes(functionConfig, typePool.describe(functionConfig.getClassName()).resolve()); } - public static Class[] getFunctionTypes(FunctionConfig functionConfig, Class functionClass) - throws ClassNotFoundException { + public static TypeDefinition[] getFunctionTypes(FunctionConfig functionConfig, TypeDefinition functionClass) { boolean isWindowConfigPresent = functionConfig.getWindowConfig() != null; return getFunctionTypes(functionClass, isWindowConfigPresent); } - public static Class[] getFunctionTypes(Class userClass, boolean isWindowConfigPresent) { + public static TypeDefinition[] getFunctionTypes(TypeDefinition userClass, boolean isWindowConfigPresent) { Class classParent = getFunctionClassParent(userClass, isWindowConfigPresent); - Class[] typeArgs = TypeResolver.resolveRawArguments(classParent, userClass); + TypeList.Generic typeArgsList = resolveInterfaceTypeArguments(userClass, classParent); + TypeDescription.Generic[] typeArgs = new TypeDescription.Generic[2]; + typeArgs[0] = typeArgsList.get(0); + typeArgs[1] = typeArgsList.get(1); // if window function if (isWindowConfigPresent) { if (classParent.equals(java.util.function.Function.class)) { - if (!typeArgs[0].equals(Collection.class)) { + if (!typeArgs[0].asErasure().isAssignableTo(Collection.class)) { throw new IllegalArgumentException("Window function must take a collection as input"); } - typeArgs[0] = (Class) unwrapType(classParent, userClass, 0); + typeArgs[0] = typeArgs[0].getTypeArguments().get(0); } } - if (typeArgs[1].equals(Record.class)) { - typeArgs[1] = (Class) unwrapType(classParent, userClass, 1); + if (typeArgs[1].asErasure().isAssignableTo(Record.class)) { + typeArgs[1] = typeArgs[1].getTypeArguments().get(0); + } + if (typeArgs[1].asErasure().isAssignableTo(CompletableFuture.class)) { + typeArgs[1] = typeArgs[1].getTypeArguments().get(0); } - return typeArgs; } - public static Class[] getRawFunctionTypes(Class userClass, boolean isWindowConfigPresent) { + private static TypeList.Generic resolveInterfaceTypeArguments(TypeDefinition userClass, Class interfaceClass) { + if (!interfaceClass.isInterface()) { + throw new IllegalArgumentException("interfaceClass must be an interface"); + } + for (TypeDescription.Generic interfaze : userClass.getInterfaces()) { + if (interfaze.asErasure().isAssignableTo(interfaceClass)) { + return interfaze.getTypeArguments(); + } + } + if (userClass.getSuperClass() != null) { + return resolveInterfaceTypeArguments(userClass.getSuperClass(), interfaceClass); + } + return null; + } + + public static TypeDescription.Generic[] getRawFunctionTypes(TypeDefinition userClass, + boolean isWindowConfigPresent) { Class classParent = getFunctionClassParent(userClass, isWindowConfigPresent); - return TypeResolver.resolveRawArguments(classParent, userClass); + TypeList.Generic typeArgsList = resolveInterfaceTypeArguments(userClass, classParent); + TypeDescription.Generic[] typeArgs = new TypeDescription.Generic[2]; + typeArgs[0] = typeArgsList.get(0); + typeArgs[1] = typeArgsList.get(1); + return typeArgs; } - public static Class getFunctionClassParent(Class userClass, boolean isWindowConfigPresent) { + public static Class getFunctionClassParent(TypeDefinition userClass, boolean isWindowConfigPresent) { if (isWindowConfigPresent) { - if (WindowFunction.class.isAssignableFrom(userClass)) { + if (userClass.asErasure().isAssignableTo(WindowFunction.class)) { return WindowFunction.class; } else { return java.util.function.Function.class; } } else { - if (Function.class.isAssignableFrom(userClass)) { + if (userClass.asErasure().isAssignableTo(Function.class)) { return Function.class; } else { return java.util.function.Function.class; @@ -148,41 +164,6 @@ public static Class getFunctionClassParent(Class userClass, boolean isWind } } - private static Type unwrapType(Class type, Class subType, int position) { - Type genericType = TypeResolver.resolveGenericType(type, subType); - Type argType = ((ParameterizedType) genericType).getActualTypeArguments()[position]; - return ((ParameterizedType) argType).getActualTypeArguments()[0]; - } - - public static Object createInstance(String userClassName, ClassLoader classLoader) { - Class theCls; - try { - theCls = Class.forName(userClassName); - } catch (ClassNotFoundException | NoClassDefFoundError cnfe) { - try { - theCls = Class.forName(userClassName, true, classLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new RuntimeException("User class must be in class path", cnfe); - } - } - Object result; - try { - Constructor meth = theCls.getDeclaredConstructor(); - meth.setAccessible(true); - result = meth.newInstance(); - } catch (InstantiationException ie) { - throw new RuntimeException("User class must be concrete", ie); - } catch (NoSuchMethodException e) { - throw new RuntimeException("User class doesn't have such method", e); - } catch (IllegalAccessException e) { - throw new RuntimeException("User class must have a no-arg constructor", e); - } catch (InvocationTargetException e) { - throw new RuntimeException("User class constructor throws exception", e); - } - return result; - - } - public static Runtime convertRuntime(FunctionConfig.Runtime runtime) { for (Runtime type : Runtime.values()) { if (type.name().equals(runtime.name())) { @@ -223,29 +204,34 @@ public static FunctionConfig.ProcessingGuarantees convertProcessingGuarantee( throw new RuntimeException("Unrecognized processing guarantee: " + processingGuarantees.name()); } - public static Class getSourceType(String className, ClassLoader classLoader) throws ClassNotFoundException { - return getSourceType(classLoader.loadClass(className)); + public static TypeDefinition getSourceType(String className, TypePool typePool) { + return getSourceType(typePool.describe(className).resolve()); } - public static Class getSourceType(Class sourceClass) { - - if (Source.class.isAssignableFrom(sourceClass)) { - return TypeResolver.resolveRawArgument(Source.class, sourceClass); - } else if (BatchSource.class.isAssignableFrom(sourceClass)) { - return TypeResolver.resolveRawArgument(BatchSource.class, sourceClass); + public static TypeDefinition getSourceType(TypeDefinition sourceClass) { + if (sourceClass.asErasure().isAssignableTo(Source.class)) { + return resolveInterfaceTypeArguments(sourceClass, Source.class).get(0); + } else if (sourceClass.asErasure().isAssignableTo(BatchSource.class)) { + return resolveInterfaceTypeArguments(sourceClass, BatchSource.class).get(0); } else { throw new IllegalArgumentException( String.format("Source class %s does not implement the correct interface", - sourceClass.getName())); + sourceClass.getActualName())); } } - public static Class getSinkType(String className, ClassLoader classLoader) throws ClassNotFoundException { - return getSinkType(classLoader.loadClass(className)); + public static TypeDefinition getSinkType(String className, TypePool typePool) { + return getSinkType(typePool.describe(className).resolve()); } - public static Class getSinkType(Class sinkClass) { - return TypeResolver.resolveRawArgument(Sink.class, sinkClass); + public static TypeDefinition getSinkType(TypeDefinition sinkClass) { + if (sinkClass.asErasure().isAssignableTo(Sink.class)) { + return resolveInterfaceTypeArguments(sinkClass, Sink.class).get(0); + } else { + throw new IllegalArgumentException( + String.format("Sink class %s does not implement the correct interface", + sinkClass.getActualName())); + } } public static void downloadFromHttpUrl(String destPkgUrl, File targetFile) throws IOException { @@ -264,16 +250,6 @@ public static void downloadFromHttpUrl(String destPkgUrl, File targetFile) throw log.info("Downloading function package from {} to {} completed!", destPkgUrl, targetFile.getAbsoluteFile()); } - public static ClassLoader extractClassLoader(String destPkgUrl) throws IOException, URISyntaxException { - File file = extractFileFromPkgURL(destPkgUrl); - try { - return ClassLoaderUtils.loadJar(file); - } catch (MalformedURLException e) { - throw new IllegalArgumentException( - "Corrupt User PackageFile " + file + " with error " + e.getMessage()); - } - } - public static File createPkgTempFile() throws IOException { return File.createTempFile("functions", ".tmp"); } @@ -297,21 +273,6 @@ public static File extractFileFromPkgURL(String destPkgUrl) throws IOException, } } - public static NarClassLoader extractNarClassLoader(File packageFile, - String narExtractionDirectory) { - if (packageFile != null) { - try { - return NarClassLoaderBuilder.builder() - .narFile(packageFile) - .extractionDirectory(narExtractionDirectory) - .build(); - } catch (IOException e) { - throw new IllegalArgumentException(e.getMessage()); - } - } - return null; - } - public static String getFullyQualifiedInstanceId(org.apache.pulsar.functions.proto.Function.Instance instance) { return getFullyQualifiedInstanceId( instance.getFunctionMetaData().getFunctionDetails().getTenant(), @@ -345,17 +306,6 @@ public static final MessageId getMessageId(long sequenceId) { return new MessageIdImpl(ledgerId, entryId, -1); } - public static byte[] toByteArray(Object obj) throws IOException { - byte[] bytes = null; - try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(bos)) { - oos.writeObject(obj); - oos.flush(); - bytes = bos.toByteArray(); - } - return bytes; - } - public static String getUniquePackageName(String packageName) { return String.format("%s-%s", UUID.randomUUID().toString(), packageName); } @@ -403,146 +353,11 @@ private static String extractFromFullyQualifiedName(String fqfn, int index) { throw new RuntimeException("Invalid Fully Qualified Function Name " + fqfn); } - public static Class getTypeArg(String className, Class funClass, ClassLoader classLoader) - throws ClassNotFoundException { - Class loadedClass = classLoader.loadClass(className); - if (!funClass.isAssignableFrom(loadedClass)) { - throw new IllegalArgumentException( - String.format("class %s is not type of %s", className, funClass.getName())); - } - return TypeResolver.resolveRawArgument(funClass, loadedClass); - } - public static double roundDecimal(double value, int places) { double scale = Math.pow(10, places); return Math.round(value * scale) / scale; } - public static ClassLoader getClassLoaderFromPackage( - ComponentType componentType, - String className, - File packageFile, - String narExtractionDirectory) { - String connectorClassName = className; - ClassLoader jarClassLoader = null; - boolean keepJarClassLoader = false; - ClassLoader narClassLoader = null; - boolean keepNarClassLoader = false; - - Exception jarClassLoaderException = null; - Exception narClassLoaderException = null; - - try { - try { - jarClassLoader = ClassLoaderUtils.extractClassLoader(packageFile); - } catch (Exception e) { - jarClassLoaderException = e; - } - try { - narClassLoader = FunctionCommon.extractNarClassLoader(packageFile, narExtractionDirectory); - } catch (Exception e) { - narClassLoaderException = e; - } - - // if connector class name is not provided, we can only try to load archive as a NAR - if (isEmpty(connectorClassName)) { - if (narClassLoader == null) { - throw new IllegalArgumentException(String.format("%s package does not have the correct format. " - + "Pulsar cannot determine if the package is a NAR package or JAR package. " - + "%s classname is not provided and attempts to load it as a NAR package produced " - + "the following error.", - capFirstLetter(componentType), capFirstLetter(componentType)), - narClassLoaderException); - } - try { - if (componentType == ComponentType.FUNCTION) { - connectorClassName = FunctionUtils.getFunctionClass(narClassLoader); - } else if (componentType == ComponentType.SOURCE) { - connectorClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) narClassLoader); - } else { - connectorClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) narClassLoader); - } - } catch (IOException e) { - throw new IllegalArgumentException(String.format("Failed to extract %s class from archive", - componentType.toString().toLowerCase()), e); - } - - try { - narClassLoader.loadClass(connectorClassName); - keepNarClassLoader = true; - return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", capFirstLetter(componentType), - connectorClassName), e); - } - - } else { - // if connector class name is provided, we need to try to load it as a JAR and as a NAR. - if (jarClassLoader != null) { - try { - jarClassLoader.loadClass(connectorClassName); - keepJarClassLoader = true; - return jarClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e) { - // class not found in JAR try loading as a NAR and searching for the class - if (narClassLoader != null) { - - try { - narClassLoader.loadClass(connectorClassName); - keepNarClassLoader = true; - return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e1) { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", - capFirstLetter(componentType), connectorClassName), e1); - } - } else { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", capFirstLetter(componentType), - connectorClassName), e); - } - } - } else if (narClassLoader != null) { - try { - narClassLoader.loadClass(connectorClassName); - keepNarClassLoader = true; - return narClassLoader; - } catch (ClassNotFoundException | NoClassDefFoundError e1) { - throw new IllegalArgumentException( - String.format("%s class %s must be in class path", - capFirstLetter(componentType), connectorClassName), e1); - } - } else { - StringBuilder errorMsg = new StringBuilder(capFirstLetter(componentType) - + " package does not have the correct format." - + " Pulsar cannot determine if the package is a NAR package or JAR package."); - - if (jarClassLoaderException != null) { - errorMsg.append( - " Attempts to load it as a JAR package produced error: " + jarClassLoaderException - .getMessage()); - } - - if (narClassLoaderException != null) { - errorMsg.append( - " Attempts to load it as a NAR package produced error: " + narClassLoaderException - .getMessage()); - } - - throw new IllegalArgumentException(errorMsg.toString()); - } - } - } finally { - if (!keepJarClassLoader) { - ClassLoaderUtils.closeClassLoader(jarClassLoader); - } - if (!keepNarClassLoader) { - ClassLoaderUtils.closeClassLoader(narClassLoader); - } - } - } - public static String capFirstLetter(Enum en) { return StringUtils.capitalize(en.toString().toLowerCase()); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index 0a0f732e948a5..a2a1764692e86 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -18,12 +18,11 @@ */ package org.apache.pulsar.functions.utils; -import static org.apache.commons.lang.StringUtils.isBlank; -import static org.apache.commons.lang.StringUtils.isNotBlank; -import static org.apache.commons.lang.StringUtils.isNotEmpty; +import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isEmpty; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.pulsar.common.functions.Utils.BUILTIN; -import static org.apache.pulsar.common.util.ClassLoaderUtils.loadJar; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromCompressionType; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsCompressionType; import static org.apache.pulsar.functions.utils.FunctionCommon.convertFromFunctionDetailsSubscriptionPosition; @@ -32,9 +31,7 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import java.io.File; -import java.io.IOException; import java.lang.reflect.Type; -import java.net.MalformedURLException; import java.util.Collection; import java.util.HashMap; import java.util.LinkedList; @@ -44,10 +41,13 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.pool.TypePool; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.functions.Resources; import org.apache.pulsar.common.functions.WindowConfig; @@ -55,7 +55,6 @@ import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.utils.functions.FunctionUtils; @Slf4j public class FunctionConfigUtils { @@ -74,26 +73,21 @@ public static class ExtractedFunctionDetails { private static final ObjectMapper OBJECT_MAPPER = ObjectMapperFactory.create(); - public static FunctionDetails convert(FunctionConfig functionConfig, ClassLoader classLoader) - throws IllegalArgumentException { + public static FunctionDetails convert(FunctionConfig functionConfig) { + return convert(functionConfig, (ValidatableFunctionPackage) null); + } - if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) { - if (classLoader != null) { - try { - Class[] typeArgs = FunctionCommon.getFunctionTypes(functionConfig, classLoader); - return convert( - functionConfig, - new ExtractedFunctionDetails( - functionConfig.getClassName(), - typeArgs[0].getName(), - typeArgs[1].getName())); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("Function class %s must be in class path", functionConfig.getClassName()), e); - } - } + public static FunctionDetails convert(FunctionConfig functionConfig, + ValidatableFunctionPackage validatableFunctionPackage) + throws IllegalArgumentException { + if (functionConfig == null) { + throw new IllegalArgumentException("Function config is not provided"); + } + if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA && validatableFunctionPackage != null) { + return convert(functionConfig, doJavaChecks(functionConfig, validatableFunctionPackage)); + } else { + return convert(functionConfig, new ExtractedFunctionDetails(functionConfig.getClassName(), null, null)); } - return convert(functionConfig, new ExtractedFunctionDetails(functionConfig.getClassName(), null, null)); } public static FunctionDetails convert(FunctionConfig functionConfig, ExtractedFunctionDetails extractedDetails) @@ -593,48 +587,49 @@ public static void inferMissingArguments(FunctionConfig functionConfig, } } - public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfig, ClassLoader clsLoader) { + public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfig, + ValidatableFunctionPackage validatableFunctionPackage) { - String functionClassName = functionConfig.getClassName(); - Class functionClass; + String functionClassName = StringUtils.trimToNull(functionConfig.getClassName()); + TypeDefinition functionClass; try { // if class name in function config is not set, this should be a built-in function // thus we should try to find its class name in the NAR service definition if (functionClassName == null) { - try { - functionClassName = FunctionUtils.getFunctionClass(clsLoader); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract source class from archive", e); + FunctionDefinition functionDefinition = + validatableFunctionPackage.getFunctionMetaData(FunctionDefinition.class); + if (functionDefinition == null) { + throw new IllegalArgumentException("Function class name is not provided."); + } + functionClassName = functionDefinition.getFunctionClass(); + if (functionClassName == null) { + throw new IllegalArgumentException("Function class name is not provided."); } } - functionClass = clsLoader.loadClass(functionClassName); + functionClass = validatableFunctionPackage.resolveType(functionClassName); - if (!org.apache.pulsar.functions.api.Function.class.isAssignableFrom(functionClass) - && !java.util.function.Function.class.isAssignableFrom(functionClass) - && !org.apache.pulsar.functions.api.WindowFunction.class.isAssignableFrom(functionClass)) { + if (!functionClass.asErasure().isAssignableTo(org.apache.pulsar.functions.api.Function.class) + && !functionClass.asErasure().isAssignableTo(java.util.function.Function.class) + && !functionClass.asErasure() + .isAssignableTo(org.apache.pulsar.functions.api.WindowFunction.class)) { throw new IllegalArgumentException( String.format("Function class %s does not implement the correct interface", - functionClass.getName())); + functionClassName)); } - } catch (ClassNotFoundException | NoClassDefFoundError e) { + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( - String.format("Function class %s must be in class path", functionConfig.getClassName()), e); + String.format("Function class %s must be in class path", functionClassName), e); } - Class[] typeArgs; - try { - typeArgs = FunctionCommon.getFunctionTypes(functionConfig, functionClass); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("Function class %s must be in class path", functionConfig.getClassName()), e); - } + TypeDefinition[] typeArgs = FunctionCommon.getFunctionTypes(functionConfig, functionClass); // inputs use default schema, so there is no check needed there // Check if the Input serialization/deserialization class exists in jar or already loaded and that it // implements SerDe class if (functionConfig.getCustomSerdeInputs() != null) { functionConfig.getCustomSerdeInputs().forEach((topicName, inputSerializer) -> { - ValidatorUtils.validateSerde(inputSerializer, typeArgs[0], clsLoader, true); + ValidatorUtils.validateSerde(inputSerializer, typeArgs[0], validatableFunctionPackage.getTypePool(), + true); }); } @@ -649,8 +644,8 @@ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfi throw new IllegalArgumentException( String.format("Topic %s has an incorrect schema Info", topicName)); } - ValidatorUtils.validateSchema(consumerConfig.getSchemaType(), typeArgs[0], clsLoader, true); - + ValidatorUtils.validateSchema(consumerConfig.getSchemaType(), typeArgs[0], + validatableFunctionPackage.getTypePool(), true); }); } @@ -665,13 +660,16 @@ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfi "Only one of schemaType or serdeClassName should be set in inputSpec"); } if (!isEmpty(conf.getSerdeClassName())) { - ValidatorUtils.validateSerde(conf.getSerdeClassName(), typeArgs[0], clsLoader, true); + ValidatorUtils.validateSerde(conf.getSerdeClassName(), typeArgs[0], + validatableFunctionPackage.getTypePool(), true); } if (!isEmpty(conf.getSchemaType())) { - ValidatorUtils.validateSchema(conf.getSchemaType(), typeArgs[0], clsLoader, true); + ValidatorUtils.validateSchema(conf.getSchemaType(), typeArgs[0], + validatableFunctionPackage.getTypePool(), true); } if (conf.getCryptoConfig() != null) { - ValidatorUtils.validateCryptoKeyReader(conf.getCryptoConfig(), clsLoader, false); + ValidatorUtils.validateCryptoKeyReader(conf.getCryptoConfig(), + validatableFunctionPackage.getTypePool(), false); } }); } @@ -679,8 +677,8 @@ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfi if (Void.class.equals(typeArgs[1])) { return new FunctionConfigUtils.ExtractedFunctionDetails( functionClassName, - typeArgs[0].getName(), - typeArgs[1].getName()); + typeArgs[0].asErasure().getTypeName(), + typeArgs[1].asErasure().getTypeName()); } // One and only one of outputSchemaType and outputSerdeClassName should be set @@ -690,22 +688,25 @@ public static ExtractedFunctionDetails doJavaChecks(FunctionConfig functionConfi } if (!isEmpty(functionConfig.getOutputSchemaType())) { - ValidatorUtils.validateSchema(functionConfig.getOutputSchemaType(), typeArgs[1], clsLoader, false); + ValidatorUtils.validateSchema(functionConfig.getOutputSchemaType(), typeArgs[1], + validatableFunctionPackage.getTypePool(), false); } if (!isEmpty(functionConfig.getOutputSerdeClassName())) { - ValidatorUtils.validateSerde(functionConfig.getOutputSerdeClassName(), typeArgs[1], clsLoader, false); + ValidatorUtils.validateSerde(functionConfig.getOutputSerdeClassName(), typeArgs[1], + validatableFunctionPackage.getTypePool(), false); } if (functionConfig.getProducerConfig() != null && functionConfig.getProducerConfig().getCryptoConfig() != null) { ValidatorUtils - .validateCryptoKeyReader(functionConfig.getProducerConfig().getCryptoConfig(), clsLoader, true); + .validateCryptoKeyReader(functionConfig.getProducerConfig().getCryptoConfig(), + validatableFunctionPackage.getTypePool(), true); } return new FunctionConfigUtils.ExtractedFunctionDetails( functionClassName, - typeArgs[0].getName(), - typeArgs[1].getName()); + typeArgs[0].asErasure().getTypeName(), + typeArgs[1].asErasure().getTypeName()); } private static void doPythonChecks(FunctionConfig functionConfig) { @@ -916,47 +917,21 @@ public static Collection collectAllInputTopics(FunctionConfig functionCo return retval; } - public static ClassLoader validate(FunctionConfig functionConfig, File functionPackageFile) { + public static void validateNonJavaFunction(FunctionConfig functionConfig) { doCommonChecks(functionConfig); - if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) { - ClassLoader classLoader; - if (functionPackageFile != null) { - try { - classLoader = loadJar(functionPackageFile); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("Corrupted Jar File", e); - } - } else if (!isEmpty(functionConfig.getJar())) { - File jarFile = new File(functionConfig.getJar()); - if (!jarFile.exists()) { - throw new IllegalArgumentException("Jar file does not exist"); - } - try { - classLoader = loadJar(jarFile); - } catch (Exception e) { - throw new IllegalArgumentException("Corrupted Jar File", e); - } - } else { - throw new IllegalArgumentException("Function Package is not provided"); - } - - doJavaChecks(functionConfig, classLoader); - return classLoader; - } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) { + if (functionConfig.getRuntime() == FunctionConfig.Runtime.GO) { doGolangChecks(functionConfig); - return null; } else if (functionConfig.getRuntime() == FunctionConfig.Runtime.PYTHON) { doPythonChecks(functionConfig); - return null; } else { throw new IllegalArgumentException("Function language runtime is either not set or cannot be determined"); } } public static ExtractedFunctionDetails validateJavaFunction(FunctionConfig functionConfig, - ClassLoader classLoader) { + ValidatableFunctionPackage validatableFunctionPackage) { doCommonChecks(functionConfig); - return doJavaChecks(functionConfig, classLoader); + return doJavaChecks(functionConfig, validatableFunctionPackage); } public static FunctionConfig validateUpdate(FunctionConfig existingConfig, FunctionConfig newConfig) { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java new file mode 100644 index 0000000000000..8224de32521fb --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionFilePackage.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.List; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.pool.TypePool; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; +import org.zeroturnaround.zip.ZipUtil; + +/** + * FunctionFilePackage is a class that represents a function package and + * implements the ValidatableFunctionPackage interface which decouples the + * function package from classloading. + */ +public class FunctionFilePackage implements AutoCloseable, ValidatableFunctionPackage { + private final File file; + private final ClassFileLocator.Compound classFileLocator; + private final TypePool typePool; + private final boolean isNar; + private final String narExtractionDirectory; + private final boolean enableClassloading; + + private ClassLoader classLoader; + + private final Object configMetadata; + + public FunctionFilePackage(File file, String narExtractionDirectory, boolean enableClassloading, + Class configClass) { + this.file = file; + boolean nonZeroFile = file.isFile() && file.length() > 0; + this.isNar = nonZeroFile ? ZipUtil.containsAnyEntry(file, + new String[] {"META-INF/services/pulsar-io.yaml", "META-INF/bundled-dependencies"}) : false; + this.narExtractionDirectory = narExtractionDirectory; + this.enableClassloading = enableClassloading; + if (isNar) { + List classpathFromArchive = null; + try { + classpathFromArchive = NarClassLoader.getClasspathFromArchive(file, narExtractionDirectory); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + List classFileLocators = new ArrayList<>(); + classFileLocators.add(ClassFileLocator.ForClassLoader.ofSystemLoader()); + for (File classpath : classpathFromArchive) { + if (classpath.exists()) { + try { + ClassFileLocator locator; + if (classpath.isDirectory()) { + locator = new ClassFileLocator.ForFolder(classpath); + } else { + locator = ClassFileLocator.ForJarFile.of(classpath); + } + classFileLocators.add(locator); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + this.classFileLocator = new ClassFileLocator.Compound(classFileLocators); + this.typePool = TypePool.Default.of(classFileLocator); + try { + this.configMetadata = FunctionUtils.getPulsarIOServiceConfig(file, configClass); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + try { + this.classFileLocator = nonZeroFile + ? new ClassFileLocator.Compound(ClassFileLocator.ForClassLoader.ofSystemLoader(), + ClassFileLocator.ForJarFile.of(file)) : + new ClassFileLocator.Compound(ClassFileLocator.ForClassLoader.ofSystemLoader()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + this.typePool = + TypePool.Default.of(classFileLocator); + this.configMetadata = null; + } + } + + public TypeDescription resolveType(String className) { + return typePool.describe(className).resolve(); + } + + public boolean isNar() { + return isNar; + } + + public File getFile() { + return file; + } + + public TypePool getTypePool() { + return typePool; + } + + @Override + public T getFunctionMetaData(Class clazz) { + return configMetadata != null ? clazz.cast(configMetadata) : null; + } + + @Override + public synchronized void close() throws IOException { + classFileLocator.close(); + if (classLoader instanceof Closeable) { + ((Closeable) classLoader).close(); + } + } + + public boolean isEnableClassloading() { + return enableClassloading; + } + + public synchronized ClassLoader getClassLoader() { + if (classLoader == null) { + classLoader = createClassLoader(); + } + return classLoader; + } + + private ClassLoader createClassLoader() { + if (enableClassloading) { + if (isNar) { + try { + return NarClassLoaderBuilder.builder() + .narFile(file) + .extractionDirectory(narExtractionDirectory) + .build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + try { + return new URLClassLoader(new java.net.URL[] {file.toURI().toURL()}, + NarClassLoader.class.getClassLoader()); + } catch (MalformedURLException e) { + throw new UncheckedIOException(e); + } + } + } else { + throw new IllegalStateException("Classloading is not enabled"); + } + } + + @Override + public String toString() { + return "FunctionFilePackage{" + + "file=" + file + + ", isNar=" + isNar + + '}'; + } +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java new file mode 100644 index 0000000000000..ed17478dd00ed --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionRuntimeCommon.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import static org.apache.commons.lang3.StringUtils.isEmpty; +import java.io.File; +import java.io.IOException; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.nar.NarClassLoaderBuilder; +import org.apache.pulsar.common.util.ClassLoaderUtils; +import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; +import org.apache.pulsar.functions.utils.io.ConnectorUtils; + +public class FunctionRuntimeCommon { + public static NarClassLoader extractNarClassLoader(File packageFile, + String narExtractionDirectory) { + if (packageFile != null) { + try { + return NarClassLoaderBuilder.builder() + .narFile(packageFile) + .extractionDirectory(narExtractionDirectory) + .build(); + } catch (IOException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } + return null; + } + + public static ClassLoader getClassLoaderFromPackage( + Function.FunctionDetails.ComponentType componentType, + String className, + File packageFile, + String narExtractionDirectory) { + String connectorClassName = className; + ClassLoader jarClassLoader = null; + boolean keepJarClassLoader = false; + NarClassLoader narClassLoader = null; + boolean keepNarClassLoader = false; + + Exception jarClassLoaderException = null; + Exception narClassLoaderException = null; + + try { + try { + jarClassLoader = ClassLoaderUtils.extractClassLoader(packageFile); + } catch (Exception e) { + jarClassLoaderException = e; + } + try { + narClassLoader = extractNarClassLoader(packageFile, narExtractionDirectory); + } catch (Exception e) { + narClassLoaderException = e; + } + + // if connector class name is not provided, we can only try to load archive as a NAR + if (isEmpty(connectorClassName)) { + if (narClassLoader == null) { + throw new IllegalArgumentException(String.format("%s package does not have the correct format. " + + "Pulsar cannot determine if the package is a NAR package or JAR package. " + + "%s classname is not provided and attempts to load it as a NAR package produced " + + "the following error.", + FunctionCommon.capFirstLetter(componentType), FunctionCommon.capFirstLetter(componentType)), + narClassLoaderException); + } + try { + if (componentType == Function.FunctionDetails.ComponentType.FUNCTION) { + connectorClassName = FunctionUtils.getFunctionClass(narClassLoader); + } else if (componentType == Function.FunctionDetails.ComponentType.SOURCE) { + connectorClassName = ConnectorUtils.getIOSourceClass(narClassLoader); + } else { + connectorClassName = ConnectorUtils.getIOSinkClass(narClassLoader); + } + } catch (IOException e) { + throw new IllegalArgumentException(String.format("Failed to extract %s class from archive", + componentType.toString().toLowerCase()), e); + } + + try { + narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; + return narClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e) { + throw new IllegalArgumentException(String.format("%s class %s must be in class path", + FunctionCommon.capFirstLetter(componentType), connectorClassName), e); + } + + } else { + // if connector class name is provided, we need to try to load it as a JAR and as a NAR. + if (jarClassLoader != null) { + try { + jarClassLoader.loadClass(connectorClassName); + keepJarClassLoader = true; + return jarClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e) { + // class not found in JAR try loading as a NAR and searching for the class + if (narClassLoader != null) { + + try { + narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; + return narClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e1) { + throw new IllegalArgumentException( + String.format("%s class %s must be in class path", + FunctionCommon.capFirstLetter(componentType), connectorClassName), e1); + } + } else { + throw new IllegalArgumentException(String.format("%s class %s must be in class path", + FunctionCommon.capFirstLetter(componentType), connectorClassName), e); + } + } + } else if (narClassLoader != null) { + try { + narClassLoader.loadClass(connectorClassName); + keepNarClassLoader = true; + return narClassLoader; + } catch (ClassNotFoundException | NoClassDefFoundError e1) { + throw new IllegalArgumentException( + String.format("%s class %s must be in class path", + FunctionCommon.capFirstLetter(componentType), connectorClassName), e1); + } + } else { + StringBuilder errorMsg = new StringBuilder(FunctionCommon.capFirstLetter(componentType) + + " package does not have the correct format." + + " Pulsar cannot determine if the package is a NAR package or JAR package."); + + if (jarClassLoaderException != null) { + errorMsg.append( + " Attempts to load it as a JAR package produced error: " + jarClassLoaderException + .getMessage()); + } + + if (narClassLoaderException != null) { + errorMsg.append( + " Attempts to load it as a NAR package produced error: " + narClassLoaderException + .getMessage()); + } + + throw new IllegalArgumentException(errorMsg.toString()); + } + } + } finally { + if (!keepJarClassLoader) { + ClassLoaderUtils.closeClassLoader(jarClassLoader); + } + if (!keepNarClassLoader) { + ClassLoaderUtils.closeClassLoader(narClassLoader); + } + } + } + +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java new file mode 100644 index 0000000000000..e27ed0eca1973 --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/LoadedFunctionPackage.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.pool.TypePool; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; + +/** + * LoadedFunctionPackage is a class that represents a function package and + * implements the ValidatableFunctionPackage interface which decouples the + * function package from classloading. This implementation is backed by + * a ClassLoader, and it is used when the function package is already loaded + * by a ClassLoader. This is the case in the LocalRunner and in some of + * the unit tests. + */ +public class LoadedFunctionPackage implements ValidatableFunctionPackage { + private final ClassLoader classLoader; + private final Object configMetadata; + private final TypePool typePool; + + public LoadedFunctionPackage(ClassLoader classLoader, Class configMetadataClass, T configMetadata) { + this.classLoader = classLoader; + this.configMetadata = configMetadata; + typePool = TypePool.Default.of( + ClassFileLocator.ForClassLoader.of(classLoader)); + } + + public LoadedFunctionPackage(ClassLoader classLoader, Class configMetadataClass) { + this.classLoader = classLoader; + if (classLoader instanceof NarClassLoader) { + try { + configMetadata = FunctionUtils.getPulsarIOServiceConfig((NarClassLoader) classLoader, + configMetadataClass); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + configMetadata = null; + } + typePool = TypePool.Default.of( + ClassFileLocator.ForClassLoader.of(classLoader)); + } + + @Override + public TypeDescription resolveType(String className) { + return typePool.describe(className).resolve(); + } + + @Override + public TypePool getTypePool() { + return typePool; + } + + @Override + public T getFunctionMetaData(Class clazz) { + return configMetadata != null ? clazz.cast(configMetadata) : null; + } + + @Override + public boolean isEnableClassloading() { + return true; + } + + @Override + public ClassLoader getClassLoader() { + return classLoader; + } +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java index 1cf07173ec6aa..a66751b4e634a 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SinkConfigUtils.java @@ -41,23 +41,23 @@ import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.pool.TypePool; import org.apache.commons.lang.StringUtils; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.functions.Resources; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.config.validation.ConfigValidation; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.utils.functions.FunctionUtils; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; @Slf4j public class SinkConfigUtils { @@ -409,8 +409,8 @@ public static SinkConfig convertFromDetails(FunctionDetails functionDetails) { } public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConfig, - ClassLoader sinkClassLoader, - ClassLoader functionClassLoader, + ValidatableFunctionPackage sinkFunction, + ValidatableFunctionPackage transformFunction, boolean validateConnectorConfig) { if (isEmpty(sinkConfig.getTenant())) { throw new IllegalArgumentException("Sink tenant cannot be null"); @@ -450,63 +450,72 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf // if class name in sink config is not set, this should be a built-in sink // thus we should try to find it class name in the NAR service definition if (sinkClassName == null) { - try { - sinkClassName = ConnectorUtils.getIOSinkClass((NarClassLoader) sinkClassLoader); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract sink class from archive", e); + ConnectorDefinition connectorDefinition = sinkFunction.getFunctionMetaData(ConnectorDefinition.class); + if (connectorDefinition == null) { + throw new IllegalArgumentException( + "Sink package doesn't contain the META-INF/services/pulsar-io.yaml file."); + } + sinkClassName = connectorDefinition.getSinkClass(); + if (sinkClassName == null) { + throw new IllegalArgumentException("Failed to extract sink class from archive"); } } // check if sink implements the correct interfaces - Class sinkClass; + TypeDefinition sinkClass; try { - sinkClass = sinkClassLoader.loadClass(sinkClassName); - } catch (ClassNotFoundException e) { + sinkClass = sinkFunction.resolveType(sinkClassName); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( - String.format("Sink class %s not found in class loader", sinkClassName), e); + String.format("Sink class %s not found", sinkClassName), e); } String functionClassName = sinkConfig.getTransformFunctionClassName(); - Class typeArg; - ClassLoader inputClassLoader; - if (functionClassLoader != null) { + TypeDefinition typeArg; + ValidatableFunctionPackage inputFunction; + if (transformFunction != null) { // if function class name in sink config is not set, this should be a built-in function // thus we should try to find it class name in the NAR service definition if (functionClassName == null) { - try { - functionClassName = FunctionUtils.getFunctionClass(functionClassLoader); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract function class from archive", e); + FunctionDefinition functionDefinition = + transformFunction.getFunctionMetaData(FunctionDefinition.class); + if (functionDefinition == null) { + throw new IllegalArgumentException( + "Function package doesn't contain the META-INF/services/pulsar-io.yaml file."); + } + functionClassName = functionDefinition.getFunctionClass(); + if (functionClassName == null) { + throw new IllegalArgumentException("Transform function class name must be set"); } } - Class functionClass; + TypeDefinition functionClass; try { - functionClass = functionClassLoader.loadClass(functionClassName); - } catch (ClassNotFoundException e) { + functionClass = transformFunction.resolveType(functionClassName); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( - String.format("Function class %s not found in class loader", functionClassName), e); + String.format("Function class %s not found", functionClassName), e); } // extract type from transform function class - if (!getRawFunctionTypes(functionClass, false)[1].equals(Record.class)) { + if (!getRawFunctionTypes(functionClass, false)[1].asErasure().isAssignableTo(Record.class)) { throw new IllegalArgumentException("Sink transform function output must be of type Record"); } typeArg = getFunctionTypes(functionClass, false)[0]; - inputClassLoader = functionClassLoader; + inputFunction = transformFunction; } else { // extract type from sink class typeArg = getSinkType(sinkClass); - inputClassLoader = sinkClassLoader; + inputFunction = sinkFunction; } if (sinkConfig.getTopicToSerdeClassName() != null) { for (String serdeClassName : sinkConfig.getTopicToSerdeClassName().values()) { - ValidatorUtils.validateSerde(serdeClassName, typeArg, inputClassLoader, true); + ValidatorUtils.validateSerde(serdeClassName, typeArg, inputFunction.getTypePool(), true); } } if (sinkConfig.getTopicToSchemaType() != null) { for (String schemaType : sinkConfig.getTopicToSchemaType().values()) { - ValidatorUtils.validateSchema(schemaType, typeArg, inputClassLoader, true); + ValidatorUtils.validateSchema(schemaType, typeArg, inputFunction.getTypePool(), true); } } @@ -519,23 +528,43 @@ public static ExtractedSinkDetails validateAndExtractDetails(SinkConfig sinkConf throw new IllegalArgumentException("Only one of serdeClassName or schemaType should be set"); } if (!isEmpty(consumerSpec.getSerdeClassName())) { - ValidatorUtils.validateSerde(consumerSpec.getSerdeClassName(), typeArg, inputClassLoader, true); + ValidatorUtils.validateSerde(consumerSpec.getSerdeClassName(), typeArg, + inputFunction.getTypePool(), true); } if (!isEmpty(consumerSpec.getSchemaType())) { - ValidatorUtils.validateSchema(consumerSpec.getSchemaType(), typeArg, inputClassLoader, true); + ValidatorUtils.validateSchema(consumerSpec.getSchemaType(), typeArg, + inputFunction.getTypePool(), true); } if (consumerSpec.getCryptoConfig() != null) { - ValidatorUtils.validateCryptoKeyReader(consumerSpec.getCryptoConfig(), inputClassLoader, false); + ValidatorUtils.validateCryptoKeyReader(consumerSpec.getCryptoConfig(), + inputFunction.getTypePool(), false); } } } - // validate user defined config if enabled and sink is loaded from NAR - if (validateConnectorConfig && sinkClassLoader instanceof NarClassLoader) { - validateSinkConfig(sinkConfig, (NarClassLoader) sinkClassLoader); + if (sinkConfig.getRetainKeyOrdering() != null + && sinkConfig.getRetainKeyOrdering() + && sinkConfig.getProcessingGuarantees() != null + && sinkConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) { + throw new IllegalArgumentException( + "When effectively once processing guarantee is specified, retain Key ordering cannot be set"); + } + + if (sinkConfig.getRetainKeyOrdering() != null && sinkConfig.getRetainKeyOrdering() + && sinkConfig.getRetainOrdering() != null && sinkConfig.getRetainOrdering()) { + throw new IllegalArgumentException("Only one of retain ordering or retain key ordering can be set"); } - return new ExtractedSinkDetails(sinkClassName, typeArg.getName(), functionClassName); + // validate user defined config if enabled and classloading is enabled + if (validateConnectorConfig) { + if (sinkFunction.isEnableClassloading()) { + validateSinkConfig(sinkConfig, sinkFunction); + } else { + log.warn("Skipping annotation based validation of sink config as classloading is disabled"); + } + } + + return new ExtractedSinkDetails(sinkClassName, typeArg.asErasure().getTypeName(), functionClassName); } public static Collection collectAllInputTopics(SinkConfig sinkConfig) { @@ -694,29 +723,13 @@ public static SinkConfig validateUpdate(SinkConfig existingConfig, SinkConfig ne return mergedConfig; } - public static void validateSinkConfig(SinkConfig sinkConfig, NarClassLoader narClassLoader) { - - if (sinkConfig.getRetainKeyOrdering() != null - && sinkConfig.getRetainKeyOrdering() - && sinkConfig.getProcessingGuarantees() != null - && sinkConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) { - throw new IllegalArgumentException( - "When effectively once processing guarantee is specified, retain Key ordering cannot be set"); - } - - if (sinkConfig.getRetainKeyOrdering() != null && sinkConfig.getRetainKeyOrdering() - && sinkConfig.getRetainOrdering() != null && sinkConfig.getRetainOrdering()) { - throw new IllegalArgumentException("Only one of retain ordering or retain key ordering can be set"); - } - + public static void validateSinkConfig(SinkConfig sinkConfig, ValidatableFunctionPackage sinkFunction) { try { - ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader); - if (defn.getSinkConfigClass() != null) { - Class configClass = Class.forName(defn.getSinkConfigClass(), true, narClassLoader); + ConnectorDefinition defn = sinkFunction.getFunctionMetaData(ConnectorDefinition.class); + if (defn != null && defn.getSinkConfigClass() != null) { + Class configClass = Class.forName(defn.getSinkConfigClass(), true, sinkFunction.getClassLoader()); validateSinkConfig(sinkConfig, configClass); } - } catch (IOException e) { - throw new IllegalArgumentException("Error validating sink config", e); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Could not find sink config class", e); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java index f3be015d73754..a6430bbea4585 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/SourceConfigUtils.java @@ -35,7 +35,9 @@ import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.pool.TypePool; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.ProducerConfig; import org.apache.pulsar.common.functions.Resources; @@ -44,13 +46,11 @@ import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.naming.TopicDomain; import org.apache.pulsar.common.naming.TopicName; -import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.config.validation.ConfigValidation; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; import org.apache.pulsar.io.core.BatchSource; import org.apache.pulsar.io.core.Source; @@ -294,7 +294,7 @@ public static SourceConfig convertFromDetails(FunctionDetails functionDetails) { } public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sourceConfig, - ClassLoader sourceClassLoader, + ValidatableFunctionPackage sourceFunction, boolean validateConnectorConfig) { if (isEmpty(sourceConfig.getTenant())) { throw new IllegalArgumentException("Source tenant cannot be null"); @@ -319,29 +319,34 @@ public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sour // if class name in source config is not set, this should be a built-in source // thus we should try to find it class name in the NAR service definition if (sourceClassName == null) { - try { - sourceClassName = ConnectorUtils.getIOSourceClass((NarClassLoader) sourceClassLoader); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to extract source class from archive", e); + ConnectorDefinition connectorDefinition = sourceFunction.getFunctionMetaData(ConnectorDefinition.class); + if (connectorDefinition == null) { + throw new IllegalArgumentException( + "Source package doesn't contain the META-INF/services/pulsar-io.yaml file."); + } + sourceClassName = connectorDefinition.getSourceClass(); + if (sourceClassName == null) { + throw new IllegalArgumentException("Failed to extract source class from archive"); } } // check if source implements the correct interfaces - Class sourceClass; + TypeDescription sourceClass; try { - sourceClass = sourceClassLoader.loadClass(sourceClassName); - } catch (ClassNotFoundException e) { + sourceClass = sourceFunction.resolveType(sourceClassName); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( String.format("Source class %s not found in class loader", sourceClassName), e); } - if (!Source.class.isAssignableFrom(sourceClass) && !BatchSource.class.isAssignableFrom(sourceClass)) { + if (!(sourceClass.asErasure().isAssignableTo(Source.class) || sourceClass.asErasure() + .isAssignableTo(BatchSource.class))) { throw new IllegalArgumentException( - String.format("Source class %s does not implement the correct interface", - sourceClass.getName())); + String.format("Source class %s does not implement the correct interface", + sourceClass.getName())); } - if (BatchSource.class.isAssignableFrom(sourceClass)) { + if (sourceClass.asErasure().isAssignableTo(BatchSource.class)) { if (sourceConfig.getBatchSourceConfig() != null) { validateBatchSourceConfig(sourceConfig.getBatchSourceConfig()); } else { @@ -352,7 +357,14 @@ public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sour } // extract type from source class - Class typeArg = getSourceType(sourceClass); + TypeDefinition typeArg; + + try { + typeArg = getSourceType(sourceClass); + } catch (Exception e) { + throw new IllegalArgumentException( + String.format("Failed to resolve type for Source class %s", sourceClassName), e); + } // Only one of serdeClassName or schemaType should be set if (!StringUtils.isEmpty(sourceConfig.getSerdeClassName()) && !StringUtils @@ -361,29 +373,30 @@ public static ExtractedSourceDetails validateAndExtractDetails(SourceConfig sour } if (!StringUtils.isEmpty(sourceConfig.getSerdeClassName())) { - ValidatorUtils.validateSerde(sourceConfig.getSerdeClassName(), typeArg, sourceClassLoader, false); + ValidatorUtils.validateSerde(sourceConfig.getSerdeClassName(), typeArg, sourceFunction.getTypePool(), + false); } if (!StringUtils.isEmpty(sourceConfig.getSchemaType())) { - ValidatorUtils.validateSchema(sourceConfig.getSchemaType(), typeArg, sourceClassLoader, false); + ValidatorUtils.validateSchema(sourceConfig.getSchemaType(), typeArg, sourceFunction.getTypePool(), + false); } if (sourceConfig.getProducerConfig() != null && sourceConfig.getProducerConfig().getCryptoConfig() != null) { ValidatorUtils - .validateCryptoKeyReader(sourceConfig.getProducerConfig().getCryptoConfig(), sourceClassLoader, - true); + .validateCryptoKeyReader(sourceConfig.getProducerConfig().getCryptoConfig(), + sourceFunction.getTypePool(), true); } - if (typeArg.equals(TypeResolver.Unknown.class)) { - throw new IllegalArgumentException( - String.format("Failed to resolve type for Source class %s", sourceClassName)); - } - - // validate user defined config if enabled and source is loaded from NAR - if (validateConnectorConfig && sourceClassLoader instanceof NarClassLoader) { - validateSourceConfig(sourceConfig, (NarClassLoader) sourceClassLoader); + // validate user defined config if enabled and classloading is enabled + if (validateConnectorConfig) { + if (sourceFunction.isEnableClassloading()) { + validateSourceConfig(sourceConfig, sourceFunction); + } else { + log.warn("Skipping annotation based validation of sink config as classloading is disabled"); + } } - return new ExtractedSourceDetails(sourceClassName, typeArg.getName()); + return new ExtractedSourceDetails(sourceClassName, typeArg.asErasure().getTypeName()); } @SneakyThrows @@ -524,15 +537,14 @@ public static void validateBatchSourceConfigUpdate(BatchSourceConfig existingCon } } - public static void validateSourceConfig(SourceConfig sourceConfig, NarClassLoader narClassLoader) { + public static void validateSourceConfig(SourceConfig sourceConfig, ValidatableFunctionPackage sourceFunction) { try { - ConnectorDefinition defn = ConnectorUtils.getConnectorDefinition(narClassLoader); - if (defn.getSourceConfigClass() != null) { - Class configClass = Class.forName(defn.getSourceConfigClass(), true, narClassLoader); + ConnectorDefinition defn = sourceFunction.getFunctionMetaData(ConnectorDefinition.class); + if (defn != null && defn.getSourceConfigClass() != null) { + Class configClass = + Class.forName(defn.getSourceConfigClass(), true, sourceFunction.getClassLoader()); validateSourceConfig(sourceConfig, configClass); } - } catch (IOException e) { - throw new IllegalArgumentException("Error validating source config", e); } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Could not find source config class"); } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java new file mode 100644 index 0000000000000..8d5aefb6f6785 --- /dev/null +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatableFunctionPackage.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.utils; + +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.pool.TypePool; + +/** + * This abstraction separates the function and connector definition from classloading, + * enabling validation without the need for classloading. It utilizes Byte Buddy for + * type and annotation resolution. + * + * The function or connector definition is directly extracted from the archive file, + * eliminating the need for classloader initialization. + * + * The getClassLoader method should only be invoked when classloading is enabled. + * Classloading is required in the LocalRunner and in the Functions worker when the + * worker is configured with the 'validateConnectorConfig' set to true. + */ +public interface ValidatableFunctionPackage { + /** + * Resolves the type description for the given class name within the function package. + */ + TypeDescription resolveType(String className); + /** + * Returns the Byte Buddy TypePool instance for the function package. + */ + TypePool getTypePool(); + /** + * Returns the function or connector definition metadata. + * Supports FunctionDefinition and ConnectorDefinition as the metadata type. + */ + T getFunctionMetaData(Class clazz); + /** + * Returns if classloading is enabled for the function package. + */ + boolean isEnableClassloading(); + /** + * Returns the classloader for the function package. The classloader is + * lazily initialized when classloading is enabled. + */ + ClassLoader getClassLoader(); +} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java index 390671c5606af..8df6a3f261a6e 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/ValidatorUtils.java @@ -18,35 +18,40 @@ */ package org.apache.pulsar.functions.utils; -import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isEmpty; -import static org.apache.commons.lang3.StringUtils.isNotBlank; import java.util.Map; import lombok.extern.slf4j.Slf4j; -import net.jodah.typetools.TypeResolver; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.pool.TypePool; import org.apache.pulsar.client.api.CryptoKeyReader; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.functions.CryptoConfig; import org.apache.pulsar.common.schema.SchemaType; -import org.apache.pulsar.common.util.ClassLoaderUtils; -import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.api.SerDe; -import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.io.core.Sink; -import org.apache.pulsar.io.core.Source; @Slf4j public class ValidatorUtils { private static final String DEFAULT_SERDE = "org.apache.pulsar.functions.api.utils.DefaultSerDe"; - public static void validateSchema(String schemaType, Class typeArg, ClassLoader clsLoader, + public static void validateSchema(String schemaType, TypeDefinition typeArg, TypePool typePool, boolean input) { if (isEmpty(schemaType) || getBuiltinSchemaType(schemaType) != null) { // If it's empty, we use the default schema and no need to validate // If it's built-in, no need to validate } else { - ClassLoaderUtils.implementsClass(schemaType, Schema.class, clsLoader); - validateSchemaType(schemaType, typeArg, clsLoader, input); + TypeDescription schemaClass = null; + try { + schemaClass = typePool.describe(schemaType).resolve(); + } catch (TypePool.Resolution.NoSuchTypeException e) { + throw new IllegalArgumentException( + String.format("The schema class %s does not exist", schemaType)); + } + if (!schemaClass.asErasure().isAssignableTo(Schema.class)) { + throw new IllegalArgumentException( + String.format("%s does not implement %s", schemaType, Schema.class.getName())); + } + validateSchemaType(schemaClass, typeArg, typePool, input); } } @@ -60,29 +65,32 @@ private static SchemaType getBuiltinSchemaType(String schemaTypeOrClassName) { } - public static void validateCryptoKeyReader(CryptoConfig conf, ClassLoader classLoader, boolean isProducer) { + public static void validateCryptoKeyReader(CryptoConfig conf, TypePool typePool, boolean isProducer) { if (isEmpty(conf.getCryptoKeyReaderClassName())) { return; } - Class cryptoClass; + String cryptoClassName = conf.getCryptoKeyReaderClassName(); + TypeDescription cryptoClass = null; try { - cryptoClass = ClassLoaderUtils.loadClass(conf.getCryptoKeyReaderClassName(), classLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { + cryptoClass = typePool.describe(cryptoClassName).resolve(); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( - String.format("The crypto key reader class %s does not exist", conf.getCryptoKeyReaderClassName())); + String.format("The crypto key reader class %s does not exist", cryptoClassName)); + } + if (!cryptoClass.asErasure().isAssignableTo(CryptoKeyReader.class)) { + throw new IllegalArgumentException( + String.format("%s does not implement %s", cryptoClassName, CryptoKeyReader.class.getName())); } - ClassLoaderUtils.implementsClass(conf.getCryptoKeyReaderClassName(), CryptoKeyReader.class, classLoader); - try { - cryptoClass.getConstructor(Map.class); - } catch (NoSuchMethodException ex) { + boolean hasConstructor = cryptoClass.getDeclaredMethods().stream() + .anyMatch(method -> method.isConstructor() && method.getParameters().size() == 1 + && method.getParameters().get(0).getType().asErasure().represents(Map.class)); + + if (!hasConstructor) { throw new IllegalArgumentException( String.format("The crypto key reader class %s does not implement the desired constructor.", conf.getCryptoKeyReaderClassName())); - - } catch (SecurityException e) { - throw new IllegalArgumentException("Failed to access crypto key reader class", e); } if (isProducer && (conf.getEncryptionKeys() == null || conf.getEncryptionKeys().length == 0)) { @@ -90,7 +98,7 @@ public static void validateCryptoKeyReader(CryptoConfig conf, ClassLoader classL } } - public static void validateSerde(String inputSerializer, Class typeArg, ClassLoader clsLoader, + public static void validateSerde(String inputSerializer, TypeDefinition typeArg, TypePool typePool, boolean deser) { if (isEmpty(inputSerializer)) { return; @@ -98,154 +106,53 @@ public static void validateSerde(String inputSerializer, Class typeArg, Class if (inputSerializer.equals(DEFAULT_SERDE)) { return; } + TypeDescription serdeClass; try { - Class serdeClass = ClassLoaderUtils.loadClass(inputSerializer, clsLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { + serdeClass = typePool.describe(inputSerializer).resolve(); + } catch (TypePool.Resolution.NoSuchTypeException e) { throw new IllegalArgumentException( String.format("The input serialization/deserialization class %s does not exist", inputSerializer)); } - ClassLoaderUtils.implementsClass(inputSerializer, SerDe.class, clsLoader); - - SerDe serDe = (SerDe) Reflections.createInstance(inputSerializer, clsLoader); - if (serDe == null) { - throw new IllegalArgumentException(String.format("The SerDe class %s does not exist", - inputSerializer)); - } - Class[] serDeTypes = TypeResolver.resolveRawArguments(SerDe.class, serDe.getClass()); - - // type inheritance information seems to be lost in generic type - // load the actual type class for verification - Class fnInputClass; - Class serdeInputClass; - try { - fnInputClass = Class.forName(typeArg.getName(), true, clsLoader); - serdeInputClass = Class.forName(serDeTypes[0].getName(), true, clsLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException("Failed to load type class", e); - } + TypeDescription.Generic serDeTypeArg = serdeClass.getInterfaces().stream() + .filter(i -> i.asErasure().isAssignableTo(SerDe.class)) + .findFirst() + .map(i -> i.getTypeArguments().get(0)) + .orElseThrow(() -> new IllegalArgumentException( + String.format("%s does not implement %s", inputSerializer, SerDe.class.getName()))); if (deser) { - if (!fnInputClass.isAssignableFrom(serdeInputClass)) { - throw new IllegalArgumentException("Serializer type mismatch " + typeArg + " vs " + serDeTypes[0]); + if (!serDeTypeArg.asErasure().isAssignableTo(typeArg.asErasure())) { + throw new IllegalArgumentException("Serializer type mismatch " + typeArg.getActualName() + " vs " + + serDeTypeArg.getActualName()); } } else { - if (!serdeInputClass.isAssignableFrom(fnInputClass)) { - throw new IllegalArgumentException("Serializer type mismatch " + typeArg + " vs " + serDeTypes[0]); + if (!serDeTypeArg.asErasure().isAssignableFrom(typeArg.asErasure())) { + throw new IllegalArgumentException("Serializer type mismatch " + typeArg.getActualName() + " vs " + + serDeTypeArg.getActualName()); } } } - private static void validateSchemaType(String schemaClassName, Class typeArg, ClassLoader clsLoader, + private static void validateSchemaType(TypeDefinition schema, TypeDefinition typeArg, TypePool typePool, boolean input) { - Schema schema = (Schema) Reflections.createInstance(schemaClassName, clsLoader); - if (schema == null) { - throw new IllegalArgumentException(String.format("The Schema class %s does not exist", - schemaClassName)); - } - Class[] schemaTypes = TypeResolver.resolveRawArguments(Schema.class, schema.getClass()); - // type inheritance information seems to be lost in generic type - // load the actual type class for verification - Class fnInputClass; - Class schemaInputClass; - try { - fnInputClass = Class.forName(typeArg.getName(), true, clsLoader); - schemaInputClass = Class.forName(schemaTypes[0].getName(), true, clsLoader); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException("Failed to load type class", e); - } + TypeDescription.Generic schemaTypeArg = schema.getInterfaces().stream() + .filter(i -> i.asErasure().isAssignableTo(Schema.class)) + .findFirst() + .map(i -> i.getTypeArguments().get(0)) + .orElse(null); if (input) { - if (!fnInputClass.isAssignableFrom(schemaInputClass)) { + if (!schemaTypeArg.asErasure().isAssignableTo(typeArg.asErasure())) { throw new IllegalArgumentException( - "Schema type mismatch " + typeArg + " vs " + schemaTypes[0]); + "Schema type mismatch " + typeArg.getActualName() + " vs " + schemaTypeArg.getActualName()); } } else { - if (!schemaInputClass.isAssignableFrom(fnInputClass)) { + if (!schemaTypeArg.asErasure().isAssignableFrom(typeArg.asErasure())) { throw new IllegalArgumentException( - "Schema type mismatch " + typeArg + " vs " + schemaTypes[0]); - } - } - } - - - public static void validateFunctionClassTypes(ClassLoader classLoader, - Function.FunctionDetails.Builder functionDetailsBuilder) { - - // validate only if classLoader is provided - if (classLoader == null) { - return; - } - - if (isBlank(functionDetailsBuilder.getClassName())) { - throw new IllegalArgumentException("Function class-name can't be empty"); - } - - // validate function class-type - Class functionClass; - try { - functionClass = classLoader.loadClass(functionDetailsBuilder.getClassName()); - } catch (ClassNotFoundException | NoClassDefFoundError e) { - throw new IllegalArgumentException( - String.format("Function class %s must be in class path", functionDetailsBuilder.getClassName()), e); - } - Class[] typeArgs = FunctionCommon.getFunctionTypes(functionClass, false); - - if (!(org.apache.pulsar.functions.api.Function.class.isAssignableFrom(functionClass)) - && !(java.util.function.Function.class.isAssignableFrom(functionClass))) { - throw new RuntimeException("User class must either be Function or java.util.Function"); - } - - if (functionDetailsBuilder.hasSource() && functionDetailsBuilder.getSource() != null - && isNotBlank(functionDetailsBuilder.getSource().getClassName())) { - try { - String sourceClassName = functionDetailsBuilder.getSource().getClassName(); - String argClassName = FunctionCommon.getTypeArg(sourceClassName, Source.class, classLoader).getName(); - functionDetailsBuilder - .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(argClassName)); - - // if sink-class not present then set same arg as source - if (!functionDetailsBuilder.hasSink() || isBlank(functionDetailsBuilder.getSink().getClassName())) { - functionDetailsBuilder - .setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(argClassName)); - } - - } catch (IllegalArgumentException ie) { - throw ie; - } catch (Exception e) { - log.error("Failed to validate source class", e); - throw new IllegalArgumentException("Failed to validate source class-name", e); - } - } else if (isBlank(functionDetailsBuilder.getSourceBuilder().getTypeClassName())) { - // if function-src-class is not present then set function-src type-class according to function class - functionDetailsBuilder - .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(typeArgs[0].getName())); - } - - if (functionDetailsBuilder.hasSink() && functionDetailsBuilder.getSink() != null - && isNotBlank(functionDetailsBuilder.getSink().getClassName())) { - try { - String sinkClassName = functionDetailsBuilder.getSink().getClassName(); - String argClassName = FunctionCommon.getTypeArg(sinkClassName, Sink.class, classLoader).getName(); - functionDetailsBuilder.setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(argClassName)); - - // if source-class not present then set same arg as sink - if (!functionDetailsBuilder.hasSource() || isBlank(functionDetailsBuilder.getSource().getClassName())) { - functionDetailsBuilder - .setSource(functionDetailsBuilder.getSourceBuilder().setTypeClassName(argClassName)); - } - - } catch (IllegalArgumentException ie) { - throw ie; - } catch (Exception e) { - log.error("Failed to validate sink class", e); - throw new IllegalArgumentException("Failed to validate sink class-name", e); + "Schema type mismatch " + typeArg.getActualName() + " vs " + schemaTypeArg.getActualName()); } - } else if (isBlank(functionDetailsBuilder.getSinkBuilder().getTypeClassName())) { - // if function-sink-class is not present then set function-sink type-class according to function class - functionDetailsBuilder - .setSink(functionDetailsBuilder.getSinkBuilder().setTypeClassName(typeArgs[1].getName())); } } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java index 028b57d69c86b..cfb213f34ed72 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionArchive.java @@ -19,14 +19,50 @@ package org.apache.pulsar.functions.utils.functions; import java.nio.file.Path; -import lombok.Builder; -import lombok.Data; import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.functions.utils.FunctionFilePackage; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; -@Builder -@Data -public class FunctionArchive { - private Path archivePath; - private ClassLoader classLoader; - private FunctionDefinition functionDefinition; +public class FunctionArchive implements AutoCloseable { + private final Path archivePath; + private final FunctionDefinition functionDefinition; + private final String narExtractionDirectory; + private final boolean enableClassloading; + private ValidatableFunctionPackage functionPackage; + private boolean closed; + + public FunctionArchive(Path archivePath, FunctionDefinition functionDefinition, String narExtractionDirectory, + boolean enableClassloading) { + this.archivePath = archivePath; + this.functionDefinition = functionDefinition; + this.narExtractionDirectory = narExtractionDirectory; + this.enableClassloading = enableClassloading; + } + + public Path getArchivePath() { + return archivePath; + } + + public synchronized ValidatableFunctionPackage getFunctionPackage() { + if (closed) { + throw new IllegalStateException("FunctionArchive is already closed"); + } + if (functionPackage == null) { + functionPackage = new FunctionFilePackage(archivePath.toFile(), narExtractionDirectory, enableClassloading, + FunctionDefinition.class); + } + return functionPackage; + } + + public FunctionDefinition getFunctionDefinition() { + return functionDefinition; + } + + @Override + public synchronized void close() throws Exception { + closed = true; + if (functionPackage instanceof AutoCloseable) { + ((AutoCloseable) functionPackage).close(); + } + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java index 941df573e495e..31a5540e0bfaf 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/functions/FunctionUtils.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pulsar.functions.utils.functions; import java.io.File; @@ -30,10 +31,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.functions.FunctionDefinition; import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarClassLoaderBuilder; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.functions.api.Function; -import org.apache.pulsar.functions.utils.Exceptions; +import org.zeroturnaround.zip.ZipUtil; @UtilityClass @@ -45,43 +44,40 @@ public class FunctionUtils { /** * Extract the Pulsar Function class from a function or archive. */ - public static String getFunctionClass(ClassLoader classLoader) throws IOException { - NarClassLoader ncl = (NarClassLoader) classLoader; - String configStr = ncl.getServiceDefinition(PULSAR_IO_SERVICE_NAME); - - FunctionDefinition conf = ObjectMapperFactory.getYamlMapper().reader().readValue(configStr, - FunctionDefinition.class); - if (StringUtils.isEmpty(conf.getFunctionClass())) { - throw new IOException( - String.format("The '%s' functionctor does not provide a function implementation", conf.getName())); - } + public static String getFunctionClass(File narFile) throws IOException { + return getFunctionDefinition(narFile).getFunctionClass(); + } - try { - // Try to load source class and check it implements Function interface - Class functionClass = ncl.loadClass(conf.getFunctionClass()); - if (!(Function.class.isAssignableFrom(functionClass))) { - throw new IOException( - "Class " + conf.getFunctionClass() + " does not implement interface " + Function.class - .getName()); - } - } catch (Throwable t) { - Exceptions.rethrowIOException(t); + public static FunctionDefinition getFunctionDefinition(File narFile) throws IOException { + return getPulsarIOServiceConfig(narFile, FunctionDefinition.class); + } + + public static T getPulsarIOServiceConfig(File narFile, Class valueType) throws IOException { + String filename = "META-INF/services/" + PULSAR_IO_SERVICE_NAME; + byte[] configEntry = ZipUtil.unpackEntry(narFile, filename); + if (configEntry != null) { + return ObjectMapperFactory.getYamlMapper().reader().readValue(configEntry, valueType); + } else { + return null; } + } - return conf.getFunctionClass(); + public static String getFunctionClass(NarClassLoader narClassLoader) throws IOException { + return getFunctionDefinition(narClassLoader).getFunctionClass(); } public static FunctionDefinition getFunctionDefinition(NarClassLoader narClassLoader) throws IOException { - String configStr = narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME); - return ObjectMapperFactory.getYamlMapper().reader().readValue(configStr, FunctionDefinition.class); + return getPulsarIOServiceConfig(narClassLoader, FunctionDefinition.class); } - public static TreeMap searchForFunctions(String functionsDirectory) throws IOException { - return searchForFunctions(functionsDirectory, false); + public static T getPulsarIOServiceConfig(NarClassLoader narClassLoader, Class valueType) throws IOException { + return ObjectMapperFactory.getYamlMapper().reader() + .readValue(narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME), valueType); } public static TreeMap searchForFunctions(String functionsDirectory, - boolean alwaysPopulatePath) throws IOException { + String narExtractionDirectory, + boolean enableClassloading) throws IOException { Path path = Paths.get(functionsDirectory).toAbsolutePath(); log.info("Searching for functions in {}", path); @@ -95,22 +91,12 @@ public static TreeMap searchForFunctions(String functio try (DirectoryStream stream = Files.newDirectoryStream(path, "*.nar")) { for (Path archive : stream) { try { - - NarClassLoader ncl = NarClassLoaderBuilder.builder() - .narFile(new File(archive.toString())) - .build(); - - FunctionArchive.FunctionArchiveBuilder functionArchiveBuilder = FunctionArchive.builder(); - FunctionDefinition cntDef = FunctionUtils.getFunctionDefinition(ncl); + FunctionDefinition cntDef = FunctionUtils.getFunctionDefinition(archive.toFile()); log.info("Found function {} from {}", cntDef, archive); - - functionArchiveBuilder.archivePath(archive); - - functionArchiveBuilder.classLoader(ncl); - functionArchiveBuilder.functionDefinition(cntDef); - - if (alwaysPopulatePath || !StringUtils.isEmpty(cntDef.getFunctionClass())) { - functions.put(cntDef.getName(), functionArchiveBuilder.build()); + if (!StringUtils.isEmpty(cntDef.getFunctionClass())) { + FunctionArchive functionArchive = + new FunctionArchive(archive, cntDef, narExtractionDirectory, enableClassloading); + functions.put(cntDef.getName(), functionArchive); } } catch (Throwable t) { log.warn("Failed to load function from {}", archive, t); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java index f1a03f4424ec6..5fcc22747c516 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/Connector.java @@ -20,17 +20,79 @@ import java.nio.file.Path; import java.util.List; -import lombok.Builder; -import lombok.Data; +import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.functions.utils.FunctionFilePackage; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; -@Builder -@Data -public class Connector { - private Path archivePath; +public class Connector implements AutoCloseable { + private final Path archivePath; + private final String narExtractionDirectory; + private final boolean enableClassloading; + private ValidatableFunctionPackage connectorFunctionPackage; private List sourceConfigFieldDefinitions; private List sinkConfigFieldDefinitions; - private ClassLoader classLoader; private ConnectorDefinition connectorDefinition; + private boolean closed; + + public Connector(Path archivePath, ConnectorDefinition connectorDefinition, String narExtractionDirectory, + boolean enableClassloading) { + this.archivePath = archivePath; + this.connectorDefinition = connectorDefinition; + this.narExtractionDirectory = narExtractionDirectory; + this.enableClassloading = enableClassloading; + } + + public Path getArchivePath() { + return archivePath; + } + + public synchronized ValidatableFunctionPackage getConnectorFunctionPackage() { + checkState(); + if (connectorFunctionPackage == null) { + connectorFunctionPackage = + new FunctionFilePackage(archivePath.toFile(), narExtractionDirectory, enableClassloading, + ConnectorDefinition.class); + } + return connectorFunctionPackage; + } + + private void checkState() { + if (closed) { + throw new IllegalStateException("Connector is already closed"); + } + } + + public synchronized List getSourceConfigFieldDefinitions() { + checkState(); + if (sourceConfigFieldDefinitions == null && !StringUtils.isEmpty(connectorDefinition.getSourceClass()) + && !StringUtils.isEmpty(connectorDefinition.getSourceConfigClass())) { + sourceConfigFieldDefinitions = ConnectorUtils.getConnectorConfigDefinition(getConnectorFunctionPackage(), + connectorDefinition.getSourceConfigClass()); + } + return sourceConfigFieldDefinitions; + } + + public synchronized List getSinkConfigFieldDefinitions() { + checkState(); + if (sinkConfigFieldDefinitions == null && !StringUtils.isEmpty(connectorDefinition.getSinkClass()) + && !StringUtils.isEmpty(connectorDefinition.getSinkConfigClass())) { + sinkConfigFieldDefinitions = ConnectorUtils.getConnectorConfigDefinition(getConnectorFunctionPackage(), + connectorDefinition.getSinkConfigClass()); + } + return sinkConfigFieldDefinitions; + } + + public ConnectorDefinition getConnectorDefinition() { + return connectorDefinition; + } + + @Override + public synchronized void close() throws Exception { + closed = true; + if (connectorFunctionPackage instanceof AutoCloseable) { + ((AutoCloseable) connectorFunctionPackage).close(); + } + } } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java index a814bf35548f3..df1310965f392 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java @@ -18,38 +18,31 @@ */ package org.apache.pulsar.functions.utils.io; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.File; import java.io.IOException; -import java.lang.annotation.Annotation; -import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.AbstractMap; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; +import net.bytebuddy.description.annotation.AnnotationDescription; +import net.bytebuddy.description.annotation.AnnotationValue; +import net.bytebuddy.description.field.FieldDescription; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDefinition; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.common.io.ConfigFieldDefinition; import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarClassLoaderBuilder; -import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.utils.Exceptions; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; import org.apache.pulsar.io.core.BatchSource; import org.apache.pulsar.io.core.Sink; import org.apache.pulsar.io.core.Source; @@ -76,7 +69,7 @@ public static String getIOSourceClass(NarClassLoader narClassLoader) throws IOEx Class sourceClass = narClassLoader.loadClass(conf.getSourceClass()); if (!(Source.class.isAssignableFrom(sourceClass) || BatchSource.class.isAssignableFrom(sourceClass))) { throw new IOException(String.format("Class %s does not implement interface %s or %s", - conf.getSourceClass(), Source.class.getName(), BatchSource.class.getName())); + conf.getSourceClass(), Source.class.getName(), BatchSource.class.getName())); } } catch (Throwable t) { Exceptions.rethrowIOException(t); @@ -109,32 +102,36 @@ public static String getIOSinkClass(NarClassLoader narClassLoader) throws IOExce return conf.getSinkClass(); } - public static ConnectorDefinition getConnectorDefinition(NarClassLoader narClassLoader) throws IOException { - String configStr = narClassLoader.getServiceDefinition(PULSAR_IO_SERVICE_NAME); + public static ConnectorDefinition getConnectorDefinition(File narFile) throws IOException { + return FunctionUtils.getPulsarIOServiceConfig(narFile, ConnectorDefinition.class); + } - return ObjectMapperFactory.getYamlMapper().reader().readValue(configStr, ConnectorDefinition.class); + public static ConnectorDefinition getConnectorDefinition(NarClassLoader narClassLoader) throws IOException { + return FunctionUtils.getPulsarIOServiceConfig(narClassLoader, ConnectorDefinition.class); } - public static List getConnectorConfigDefinition(ClassLoader classLoader, - String configClassName) throws Exception { + public static List getConnectorConfigDefinition( + ValidatableFunctionPackage connectorFunctionPackage, + String configClassName) { List retval = new LinkedList<>(); - Class configClass = classLoader.loadClass(configClassName); - for (Field field : Reflections.getAllFields(configClass)) { - if (java.lang.reflect.Modifier.isStatic(field.getModifiers())) { - // We dont want static fields + TypeDefinition configClass = connectorFunctionPackage.resolveType(configClassName); + + for (FieldDescription field : getAllFields(configClass)) { + if (field.isStatic()) { + // We don't want static fields continue; } - field.setAccessible(true); ConfigFieldDefinition configFieldDefinition = new ConfigFieldDefinition(); configFieldDefinition.setFieldName(field.getName()); - configFieldDefinition.setTypeName(field.getType().getName()); + configFieldDefinition.setTypeName(field.getType().getActualName()); Map attributes = new HashMap<>(); - for (Annotation annotation : field.getAnnotations()) { - if (annotation.annotationType().equals(FieldDoc.class)) { - FieldDoc fieldDoc = (FieldDoc) annotation; - for (Method method : FieldDoc.class.getDeclaredMethods()) { - Object value = method.invoke(fieldDoc); - attributes.put(method.getName(), value == null ? "" : value.toString()); + for (AnnotationDescription annotation : field.getDeclaredAnnotations()) { + if (annotation.getAnnotationType().represents(FieldDoc.class)) { + for (MethodDescription.InDefinedShape method : annotation.getAnnotationType() + .getDeclaredMethods()) { + AnnotationValue value = annotation.getValue(method.getName()); + attributes.put(method.getName(), + value == null || value.resolve() == null ? "" : value.resolve().toString()); } } } @@ -145,86 +142,42 @@ public static List getConnectorConfigDefinition(ClassLoad return retval; } + private static List getAllFields(TypeDefinition type) { + List fields = new LinkedList<>(); + fields.addAll(type.getDeclaredFields()); + + if (type.getSuperClass() != null) { + fields.addAll(getAllFields(type.getSuperClass())); + } + + return fields; + } + public static TreeMap searchForConnectors(String connectorsDirectory, - String narExtractionDirectory) throws IOException { + String narExtractionDirectory, + boolean enableClassloading) throws IOException { Path path = Paths.get(connectorsDirectory).toAbsolutePath(); log.info("Searching for connectors in {}", path); + TreeMap connectors = new TreeMap<>(); + if (!path.toFile().exists()) { log.warn("Connectors archive directory not found"); - return new TreeMap<>(); + return connectors; } - List archives = new ArrayList<>(); try (DirectoryStream stream = Files.newDirectoryStream(path, "*.nar")) { for (Path archive : stream) { - archives.add(archive); - } - } - if (archives.isEmpty()) { - return new TreeMap<>(); - } - - ExecutorService oneTimeExecutor = null; - try { - int nThreads = Math.min(Runtime.getRuntime().availableProcessors(), archives.size()); - log.info("Loading {} connector definitions with a thread pool of size {}", archives.size(), nThreads); - oneTimeExecutor = Executors.newFixedThreadPool(nThreads, - new ThreadFactoryBuilder().setNameFormat("connector-extraction-executor-%d").build()); - List>> futures = new ArrayList<>(); - for (Path archive : archives) { - CompletableFuture> future = CompletableFuture.supplyAsync(() -> - getConnectorDefinitionEntry(archive, narExtractionDirectory), oneTimeExecutor); - futures.add(future); - } - - FutureUtil.waitForAll(futures).join(); - return futures.stream() - .map(CompletableFuture::join) - .filter(entry -> entry != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> a, TreeMap::new)); - } finally { - if (oneTimeExecutor != null) { - oneTimeExecutor.shutdown(); - } - } - } - - private static Map.Entry getConnectorDefinitionEntry(Path archive, - String narExtractionDirectory) { - try { - - NarClassLoader ncl = NarClassLoaderBuilder.builder() - .narFile(new File(archive.toString())) - .extractionDirectory(narExtractionDirectory) - .build(); - - Connector.ConnectorBuilder connectorBuilder = Connector.builder(); - ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(ncl); - log.info("Found connector {} from {}", cntDef, archive); - - connectorBuilder.archivePath(archive); - if (!StringUtils.isEmpty(cntDef.getSourceClass())) { - if (!StringUtils.isEmpty(cntDef.getSourceConfigClass())) { - connectorBuilder.sourceConfigFieldDefinitions( - ConnectorUtils.getConnectorConfigDefinition(ncl, - cntDef.getSourceConfigClass())); + try { + ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(archive.toFile()); + log.info("Found connector {} from {}", cntDef, archive); + Connector connector = new Connector(archive, cntDef, narExtractionDirectory, enableClassloading); + connectors.put(cntDef.getName(), connector); + } catch (Throwable t) { + log.warn("Failed to load connector from {}", archive, t); } } - - if (!StringUtils.isEmpty(cntDef.getSinkClass())) { - if (!StringUtils.isEmpty(cntDef.getSinkConfigClass())) { - connectorBuilder.sinkConfigFieldDefinitions( - ConnectorUtils.getConnectorConfigDefinition(ncl, cntDef.getSinkConfigClass())); - } - } - - connectorBuilder.classLoader(ncl); - connectorBuilder.connectorDefinition(cntDef); - return new AbstractMap.SimpleEntry(cntDef.getName(), connectorBuilder.build()); - } catch (Throwable t) { - log.warn("Failed to load connector from {}", archive, t); - return null; } + return connectors; } } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java index 131f153b08d68..90fdd4da777d3 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionCommonTest.java @@ -30,16 +30,17 @@ import com.github.tomakehurst.wiremock.WireMockServer; import java.io.File; import java.util.Collection; +import java.util.concurrent.CompletableFuture; import lombok.Cleanup; +import net.bytebuddy.description.type.TypeDefinition; +import net.bytebuddy.pool.TypePool; import org.apache.pulsar.client.impl.MessageIdImpl; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.api.Function; import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.api.WindowContext; import org.apache.pulsar.functions.api.WindowFunction; import org.assertj.core.util.Files; -import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -47,41 +48,6 @@ * Unit test of {@link Exceptions}. */ public class FunctionCommonTest { - - @Test - public void testValidateLocalFileUrl() throws Exception { - String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - try { - // eg: fileLocation : /dir/fileName.jar (invalid) - FunctionCommon.extractClassLoader(fileLocation); - Assert.fail("should fail with invalid url: without protocol"); - } catch (IllegalArgumentException ie) { - // Ok.. expected exception - } - String fileLocationWithProtocol = "file://" + fileLocation; - // eg: fileLocation : file:///dir/fileName.jar (valid) - FunctionCommon.extractClassLoader(fileLocationWithProtocol); - // eg: fileLocation : file:/dir/fileName.jar (valid) - fileLocationWithProtocol = "file:" + fileLocation; - FunctionCommon.extractClassLoader(fileLocationWithProtocol); - } - - @Test - public void testValidateHttpFileUrl() throws Exception { - - String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; - FunctionCommon.extractClassLoader(jarHttpUrl); - - jarHttpUrl = "http://_invalidurl_.com"; - try { - // eg: fileLocation : /dir/fileName.jar (invalid) - FunctionCommon.extractClassLoader(jarHttpUrl); - Assert.fail("should fail with invalid url: without protocol"); - } catch (Exception ie) { - // Ok.. expected exception - } - } - @Test public void testDownloadFile() throws Exception { final String jarHttpUrl = "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; @@ -150,6 +116,14 @@ public Record process(String input, Context context) throws Exception { } }, false }, + { + new Function>() { + @Override + public CompletableFuture process(String input, Context context) throws Exception { + return null; + } + }, false + }, { new java.util.function.Function() { @Override @@ -166,6 +140,14 @@ public Record apply(String s) { } }, false }, + { + new java.util.function.Function>() { + @Override + public CompletableFuture apply(String s) { + return null; + } + }, false + }, { new WindowFunction() { @Override @@ -182,6 +164,14 @@ public Record process(Collection> input, WindowContext c } }, true }, + { + new WindowFunction>() { + @Override + public CompletableFuture process(Collection> input, WindowContext context) throws Exception { + return null; + } + }, true + }, { new java.util.function.Function, Integer>() { @Override @@ -197,15 +187,26 @@ public Record apply(Collection strings) { return null; } }, true + }, + { + new java.util.function.Function, CompletableFuture>() { + @Override + public CompletableFuture apply(Collection strings) { + return null; + } + }, true } }; } @Test(dataProvider = "function") public void testGetFunctionTypes(Object function, boolean isWindowConfigPresent) { - Class[] types = FunctionCommon.getFunctionTypes(function.getClass(), isWindowConfigPresent); + TypePool typePool = TypePool.Default.of(function.getClass().getClassLoader()); + TypeDefinition[] types = + FunctionCommon.getFunctionTypes(typePool.describe(function.getClass().getName()).resolve(), + isWindowConfigPresent); assertEquals(types.length, 2); - assertEquals(types[0], String.class); - assertEquals(types[1], Integer.class); + assertEquals(types[0].asErasure().getTypeName(), String.class.getName()); + assertEquals(types[1].asErasure().getTypeName(), Integer.class.getName()); } } diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java index 8f46199e8ffd5..954eef44a7366 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/FunctionConfigUtilsTest.java @@ -18,13 +18,24 @@ */ package org.apache.pulsar.functions.utils; +import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; +import static org.apache.pulsar.common.functions.FunctionConfig.Runtime.PYTHON; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; import com.google.gson.Gson; - -import org.apache.pulsar.client.api.CompressionType; -import org.apache.pulsar.client.api.SubscriptionInitialPosition; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.client.api.CompressionType; +import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; @@ -32,28 +43,29 @@ import org.apache.pulsar.common.functions.Resources; import org.apache.pulsar.common.functions.WindowConfig; import org.apache.pulsar.common.util.Reflections; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.functions.api.WindowContext; +import org.apache.pulsar.functions.api.WindowFunction; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.testng.annotations.Test; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; -import static org.apache.pulsar.common.functions.FunctionConfig.Runtime.PYTHON; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertThrows; -import static org.testng.Assert.assertTrue; - /** * Unit test of {@link Reflections}. */ @Slf4j public class FunctionConfigUtilsTest { + public static class WordCountWindowFunction implements WindowFunction { + @Override + public Void process(Collection> inputs, WindowContext context) throws Exception { + for (Record input : inputs) { + Arrays.asList(input.getValue().split("\\.")).forEach(word -> context.incrCounter(word, 1)); + } + return null; + } + } + @Test public void testAutoAckConvertFailed() { @@ -63,7 +75,7 @@ public void testAutoAckConvertFailed() { functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATMOST_ONCE); assertThrows(IllegalArgumentException.class, () -> { - FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + FunctionConfigUtils.convert(functionConfig); }); } @@ -99,7 +111,7 @@ public void testConvertBackFidelity() { producerConfig.setBatchBuilder("DEFAULT"); producerConfig.setCompressionType(CompressionType.ZLIB); functionConfig.setProducerConfig(producerConfig); - Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); // add default resources @@ -119,7 +131,7 @@ public void testConvertWindow() { functionConfig.setNamespace("test-namespace"); functionConfig.setName("test-function"); functionConfig.setParallelism(1); - functionConfig.setClassName(IdentityFunction.class.getName()); + functionConfig.setClassName(WordCountWindowFunction.class.getName()); Map inputSpecs = new HashMap<>(); inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").build()); functionConfig.setInputSpecs(inputSpecs); @@ -141,7 +153,7 @@ public void testConvertWindow() { producerConfig.setBatchBuilder("KEY_BASED"); producerConfig.setCompressionType(CompressionType.SNAPPY); functionConfig.setProducerConfig(producerConfig); - Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); // WindowsFunction guarantees convert to FunctionGuarantees. @@ -163,7 +175,7 @@ public void testConvertBatchBuilder() { FunctionConfig functionConfig = createFunctionConfig(); functionConfig.setBatchBuilder("KEY_BASED"); - Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); assertEquals(functionDetails.getSink().getProducerSpec().getBatchBuilder(), "KEY_BASED"); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); @@ -519,7 +531,6 @@ private FunctionConfig createFunctionConfig() { functionConfig.setUserConfig(new HashMap<>()); functionConfig.setAutoAck(true); functionConfig.setTimeoutMs(2000L); - functionConfig.setWindowConfig(new WindowConfig().setWindowLengthCount(10)); functionConfig.setCleanupSubscription(true); functionConfig.setRuntimeFlags("-Dfoo=bar"); return functionConfig; @@ -553,7 +564,7 @@ public void testDisableForwardSourceMessageProperty() throws InvalidProtocolBuff config.setForwardSourceMessageProperty(true); FunctionConfigUtils.inferMissingArguments(config, false); assertNull(config.getForwardSourceMessageProperty()); - FunctionDetails details = FunctionConfigUtils.convert(config, FunctionConfigUtilsTest.class.getClassLoader()); + FunctionDetails details = FunctionConfigUtils.convert(config); assertFalse(details.getSink().getForwardSourceMessageProperty()); String detailsJson = "'" + JsonFormat.printer().omittingInsignificantWhitespace().print(details) + "'"; log.info("Function details : {}", detailsJson); @@ -640,7 +651,7 @@ public void testMergeDifferentOutputSchemaTypes() { @Test public void testPoolMessages() { FunctionConfig functionConfig = createFunctionConfig(); - Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig); assertFalse(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages()); FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); assertFalse(convertedConfig.getInputSpecs().get("test-input").isPoolMessages()); @@ -650,7 +661,7 @@ public void testPoolMessages() { .poolMessages(true).build()); functionConfig.setInputSpecs(inputSpecs); - functionDetails = FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + functionDetails = FunctionConfigUtils.convert(functionConfig); assertTrue(functionDetails.getSource().getInputSpecsMap().get("test-input").getPoolMessages()); convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails); diff --git a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java index 8ac9b61e3f60f..14cd77f60ff95 100644 --- a/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java +++ b/pulsar-functions/utils/src/test/java/org/apache/pulsar/functions/utils/SinkConfigUtilsTest.java @@ -18,37 +18,38 @@ */ package org.apache.pulsar.functions.utils; +import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE; +import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATMOST_ONCE; +import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.expectThrows; import com.google.common.collect.Lists; import com.google.gson.Gson; +import java.io.IOException; +import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.Accessors; import org.apache.pulsar.common.functions.ConsumerConfig; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.Resources; +import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.config.validation.ConfigValidationAnnotations; -import org.apache.pulsar.functions.api.utils.IdentityFunction; +import org.apache.pulsar.functions.api.Record; import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.io.core.Sink; +import org.apache.pulsar.io.core.SinkContext; import org.testng.annotations.Test; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE; -import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.ATMOST_ONCE; -import static org.apache.pulsar.common.functions.FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertThrows; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.expectThrows; - /** * Unit test of {@link SinkConfigUtilsTest}. */ @@ -62,6 +63,27 @@ public static class TestSinkConfig { private String configParameter; } + + public static class NopSink implements Sink { + + @Override + public void open(Map config, SinkContext sinkContext) throws Exception { + + } + + @Override + public void write(Record record) throws Exception { + + } + + @Override + public void close() throws Exception { + + } + } + + + @Test public void testAutoAckConvertFailed() throws IOException { @@ -521,7 +543,7 @@ private SinkConfig createSinkConfig() { sinkConfig.setNamespace("test-namespace"); sinkConfig.setName("test-sink"); sinkConfig.setParallelism(1); - sinkConfig.setClassName(IdentityFunction.class.getName()); + sinkConfig.setClassName(NopSink.class.getName()); Map inputSpecs = new HashMap<>(); inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").build()); sinkConfig.setInputSpecs(inputSpecs); @@ -577,13 +599,16 @@ public void testAllowDisableSinkTimeout() { SinkConfig sinkConfig = createSinkConfig(); sinkConfig.setInputSpecs(null); sinkConfig.setTopicsPattern("my-topic-*"); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, + LoadedFunctionPackage validatableFunction = + new LoadedFunctionPackage(this.getClass().getClassLoader(), ConnectorDefinition.class); + + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, null, true); sinkConfig.setTimeoutMs(null); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, null, true); sinkConfig.setTimeoutMs(0L); - SinkConfigUtils.validateAndExtractDetails(sinkConfig, this.getClass().getClassLoader(), null, + SinkConfigUtils.validateAndExtractDetails(sinkConfig, validatableFunction, null, true); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index c587a8a734888..6c794317cf10d 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -70,6 +70,7 @@ import org.apache.pulsar.functions.utils.Actions; import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SourceConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.io.Connector; @Data @@ -526,7 +527,7 @@ private File getBuiltinArchive(FunctionDetails.ComponentType componentType, Func builder.setClassName(sourceClass); functionDetails.setSource(builder); - fillSourceTypeClass(functionDetails, connector.getClassLoader(), sourceClass); + fillSourceTypeClass(functionDetails, connector.getConnectorFunctionPackage(), sourceClass); return archive; } } @@ -542,7 +543,7 @@ private File getBuiltinArchive(FunctionDetails.ComponentType componentType, Func builder.setClassName(sinkClass); functionDetails.setSink(builder); - fillSinkTypeClass(functionDetails, connector.getClassLoader(), sinkClass); + fillSinkTypeClass(functionDetails, connector.getConnectorFunctionPackage(), sinkClass); return archive; } } @@ -556,8 +557,8 @@ private File getBuiltinArchive(FunctionDetails.ComponentType componentType, Func } private void fillSourceTypeClass(FunctionDetails.Builder functionDetails, - ClassLoader narClassLoader, String className) throws ClassNotFoundException { - String typeArg = getSourceType(className, narClassLoader).getName(); + ValidatableFunctionPackage functionPackage, String className) { + String typeArg = getSourceType(className, functionPackage.getTypePool()).asErasure().getName(); SourceSpec.Builder sourceBuilder = SourceSpec.newBuilder(functionDetails.getSource()); sourceBuilder.setTypeClassName(typeArg); @@ -572,8 +573,8 @@ private void fillSourceTypeClass(FunctionDetails.Builder functionDetails, } private void fillSinkTypeClass(FunctionDetails.Builder functionDetails, - ClassLoader narClassLoader, String className) throws ClassNotFoundException { - String typeArg = getSinkType(className, narClassLoader).getName(); + ValidatableFunctionPackage functionPackage, String className) { + String typeArg = getSinkType(className, functionPackage.getTypePool()).asErasure().getName(); SinkSpec.Builder sinkBuilder = SinkSpec.newBuilder(functionDetails.getSink()); sinkBuilder.setTypeClassName(typeArg); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java index 8a9ea22c53015..f9f2738828be7 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java @@ -658,6 +658,14 @@ public void stop() { if (statsUpdater != null) { statsUpdater.shutdownNow(); } + + if (null != functionsManager) { + functionsManager.close(); + } + + if (null != connectorsManager) { + connectorsManager.close(); + } } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 6bde2b4fc8819..5e105f7057e33 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -100,6 +100,7 @@ import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.FunctionConfigUtils; import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; @@ -1788,12 +1789,6 @@ private void internalProcessFunctionRequest(final String tenant, final String na } } - protected ClassLoader getClassLoaderFromPackage(String className, - File packageFile, - String narExtractionDirectory) { - return FunctionCommon.getClassLoaderFromPackage(componentType, className, packageFile, narExtractionDirectory); - } - static File downloadPackageFile(PulsarWorkerService worker, String packageName) throws IOException, PulsarAdminException { Path tempDirectory; @@ -1863,7 +1858,7 @@ protected File getPackageFile(String functionPkgUrl) throws IOException, PulsarA } } - protected ClassLoader getBuiltinFunctionClassLoader(String archive) { + protected ValidatableFunctionPackage getBuiltinFunctionPackage(String archive) { if (!StringUtils.isEmpty(archive)) { if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { archive = archive.replaceFirst("^builtin://", ""); @@ -1873,7 +1868,7 @@ protected ClassLoader getBuiltinFunctionClassLoader(String archive) { if (function == null) { throw new IllegalArgumentException("Built-in " + componentType + " is not available"); } - return function.getClassLoader(); + return function.getFunctionPackage(); } } return null; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 078c47524f8e9..6b81d2c4918a6 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -47,7 +47,6 @@ import org.apache.pulsar.common.functions.WorkerInfo; import org.apache.pulsar.common.policies.data.ExceptionInformation; import org.apache.pulsar.common.policies.data.FunctionStatus; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.auth.FunctionAuthData; import org.apache.pulsar.functions.instance.InstanceUtils; @@ -55,11 +54,14 @@ import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.FunctionFilePackage; import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.functions.FunctionArchive; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.FunctionsManager; import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Functions; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -732,11 +734,12 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant functionConfig.setTenant(tenant); functionConfig.setNamespace(namespace); functionConfig.setName(componentName); + WorkerConfig workerConfig = worker().getWorkerConfig(); FunctionConfigUtils.inferMissingArguments( - functionConfig, worker().getWorkerConfig().isForwardSourceMessageProperty()); + functionConfig, workerConfig.isForwardSourceMessageProperty()); String archive = functionConfig.getJar(); - ClassLoader classLoader = null; + ValidatableFunctionPackage functionPackage = null; // check if function is builtin and extract classloader if (!StringUtils.isEmpty(archive)) { if (archive.startsWith(org.apache.pulsar.common.functions.Utils.BUILTIN)) { @@ -749,35 +752,38 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant if (function == null) { throw new IllegalArgumentException(String.format("No Function %s found", archive)); } - classLoader = function.getClassLoader(); + functionPackage = function.getFunctionPackage(); } } - boolean shouldCloseClassLoader = false; + boolean shouldCloseFunctionPackage = false; try { - if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) { // if function is not builtin, attempt to extract classloader from package file if it exists - if (classLoader == null && componentPackageFile != null) { - classLoader = getClassLoaderFromPackage(functionConfig.getClassName(), - componentPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); - shouldCloseClassLoader = true; + if (functionPackage == null && componentPackageFile != null) { + functionPackage = + new FunctionFilePackage(componentPackageFile, workerConfig.getNarExtractionDirectory(), + workerConfig.getEnableClassloadingOfExternalFiles(), FunctionDefinition.class); + shouldCloseFunctionPackage = true; } - if (classLoader == null) { + if (functionPackage == null) { throw new IllegalArgumentException("Function package is not provided"); } FunctionConfigUtils.ExtractedFunctionDetails functionDetails = FunctionConfigUtils.validateJavaFunction( - functionConfig, classLoader); + functionConfig, functionPackage); return FunctionConfigUtils.convert(functionConfig, functionDetails); } else { - classLoader = FunctionConfigUtils.validate(functionConfig, componentPackageFile); - shouldCloseClassLoader = true; - return FunctionConfigUtils.convert(functionConfig, classLoader); + FunctionConfigUtils.validateNonJavaFunction(functionConfig); + return FunctionConfigUtils.convert(functionConfig); } } finally { - if (shouldCloseClassLoader) { - ClassLoaderUtils.closeClassLoader(classLoader); + if (shouldCloseFunctionPackage && functionPackage instanceof AutoCloseable) { + try { + ((AutoCloseable) functionPackage).close(); + } catch (Exception e) { + log.error("Failed to close function file", e); + } } } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index c382ec9e01b35..51d1333a79c36 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -47,18 +47,20 @@ import org.apache.pulsar.common.io.SinkConfig; import org.apache.pulsar.common.policies.data.ExceptionInformation; import org.apache.pulsar.common.policies.data.SinkStatus; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.auth.FunctionAuthData; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; +import org.apache.pulsar.functions.utils.FunctionFilePackage; import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; import org.apache.pulsar.functions.utils.SinkConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Sinks; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -704,7 +706,7 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant sinkConfig.setName(sinkName); org.apache.pulsar.common.functions.Utils.inferMissingArguments(sinkConfig); - ClassLoader classLoader = null; + ValidatableFunctionPackage connectorFunctionPackage = null; // check if sink is builtin and extract classloader if (!StringUtils.isEmpty(sinkConfig.getArchive())) { String archive = sinkConfig.getArchive(); @@ -716,45 +718,62 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant if (connector == null) { throw new IllegalArgumentException("Built-in sink is not available"); } - classLoader = connector.getClassLoader(); + connectorFunctionPackage = connector.getConnectorFunctionPackage(); } } - boolean shouldCloseClassLoader = false; + boolean shouldCloseFunctionPackage = false; + ValidatableFunctionPackage transformFunctionPackage = null; + boolean shouldCloseTransformFunctionPackage = false; try { // if sink is not builtin, attempt to extract classloader from package file if it exists - if (classLoader == null && sinkPackageFile != null) { - classLoader = getClassLoaderFromPackage(sinkConfig.getClassName(), - sinkPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); - shouldCloseClassLoader = true; + WorkerConfig workerConfig = worker().getWorkerConfig(); + if (connectorFunctionPackage == null && sinkPackageFile != null) { + connectorFunctionPackage = + new FunctionFilePackage(sinkPackageFile, workerConfig.getNarExtractionDirectory(), + workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class); + shouldCloseFunctionPackage = true; } - if (classLoader == null) { + if (connectorFunctionPackage == null) { throw new IllegalArgumentException("Sink package is not provided"); } - ClassLoader functionClassLoader = null; if (isNotBlank(sinkConfig.getTransformFunction())) { - functionClassLoader = - getBuiltinFunctionClassLoader(sinkConfig.getTransformFunction()); - if (functionClassLoader == null) { + transformFunctionPackage = + getBuiltinFunctionPackage(sinkConfig.getTransformFunction()); + if (transformFunctionPackage == null) { File functionPackageFile = getPackageFile(sinkConfig.getTransformFunction()); - functionClassLoader = getClassLoaderFromPackage(sinkConfig.getTransformFunctionClassName(), - functionPackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); + transformFunctionPackage = + new FunctionFilePackage(functionPackageFile, workerConfig.getNarExtractionDirectory(), + workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class); + shouldCloseTransformFunctionPackage = true; } - if (functionClassLoader == null) { + if (transformFunctionPackage == null) { throw new IllegalArgumentException("Transform Function package not found"); } } - SinkConfigUtils.ExtractedSinkDetails sinkDetails = SinkConfigUtils.validateAndExtractDetails( - sinkConfig, classLoader, functionClassLoader, worker().getWorkerConfig().getValidateConnectorConfig()); + SinkConfigUtils.ExtractedSinkDetails sinkDetails = + SinkConfigUtils.validateAndExtractDetails(sinkConfig, connectorFunctionPackage, + transformFunctionPackage, workerConfig.getValidateConnectorConfig()); return SinkConfigUtils.convert(sinkConfig, sinkDetails); } finally { - if (shouldCloseClassLoader) { - ClassLoaderUtils.closeClassLoader(classLoader); + if (shouldCloseFunctionPackage && connectorFunctionPackage instanceof AutoCloseable) { + try { + ((AutoCloseable) connectorFunctionPackage).close(); + } catch (Exception e) { + log.error("Failed to connector function file", e); + } + } + if (shouldCloseTransformFunctionPackage && transformFunctionPackage instanceof AutoCloseable) { + try { + ((AutoCloseable) transformFunctionPackage).close(); + } catch (Exception e) { + log.error("Failed to close transform function file", e); + } } } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index c55ddf48b06b9..dea69698dd28d 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -46,18 +46,20 @@ import org.apache.pulsar.common.io.SourceConfig; import org.apache.pulsar.common.policies.data.ExceptionInformation; import org.apache.pulsar.common.policies.data.SourceStatus; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.auth.FunctionAuthData; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.utils.ComponentTypeUtils; +import org.apache.pulsar.functions.utils.FunctionFilePackage; import org.apache.pulsar.functions.utils.FunctionMetaDataUtils; import org.apache.pulsar.functions.utils.SourceConfigUtils; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; import org.apache.pulsar.functions.utils.io.Connector; import org.apache.pulsar.functions.worker.FunctionMetaDataManager; import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.service.api.Sources; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -663,7 +665,7 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant sourceConfig.setName(sourceName); org.apache.pulsar.common.functions.Utils.inferMissingArguments(sourceConfig); - ClassLoader classLoader = null; + ValidatableFunctionPackage connectorFunctionPackage = null; // check if source is builtin and extract classloader if (!StringUtils.isEmpty(sourceConfig.getArchive())) { String archive = sourceConfig.getArchive(); @@ -675,30 +677,37 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant if (connector == null) { throw new IllegalArgumentException("Built-in source is not available"); } - classLoader = connector.getClassLoader(); + connectorFunctionPackage = connector.getConnectorFunctionPackage(); } } - boolean shouldCloseClassLoader = false; + boolean shouldCloseFunctionPackage = false; try { // if source is not builtin, attempt to extract classloader from package file if it exists - if (classLoader == null && sourcePackageFile != null) { - classLoader = getClassLoaderFromPackage(sourceConfig.getClassName(), - sourcePackageFile, worker().getWorkerConfig().getNarExtractionDirectory()); - shouldCloseClassLoader = true; + WorkerConfig workerConfig = worker().getWorkerConfig(); + if (connectorFunctionPackage == null && sourcePackageFile != null) { + connectorFunctionPackage = + new FunctionFilePackage(sourcePackageFile, workerConfig.getNarExtractionDirectory(), + workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class); + shouldCloseFunctionPackage = true; } - if (classLoader == null) { + if (connectorFunctionPackage == null) { throw new IllegalArgumentException("Source package is not provided"); } SourceConfigUtils.ExtractedSourceDetails sourceDetails = SourceConfigUtils.validateAndExtractDetails( - sourceConfig, classLoader, worker().getWorkerConfig().getValidateConnectorConfig()); + sourceConfig, connectorFunctionPackage, + workerConfig.getValidateConnectorConfig()); return SourceConfigUtils.convert(sourceConfig, sourceDetails); } finally { - if (shouldCloseClassLoader) { - ClassLoaderUtils.closeClassLoader(classLoader); + if (shouldCloseFunctionPackage && connectorFunctionPackage instanceof AutoCloseable) { + try { + ((AutoCloseable) connectorFunctionPackage).close(); + } catch (Exception e) { + log.error("Failed to connector function file", e); + } } } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java index cfe087c78406a..73a229893e5d2 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImplTest.java @@ -381,6 +381,6 @@ public static FunctionConfig createDefaultFunctionConfig() { public static Function.FunctionDetails createDefaultFunctionDetails() { FunctionConfig functionConfig = createDefaultFunctionConfig(); - return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + return FunctionConfigUtils.convert(functionConfig); } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index 32a104c576993..a8415919c119b 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -18,1125 +18,82 @@ */ package org.apache.pulsar.functions.worker.rest.api.v2; - -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static org.apache.pulsar.functions.utils.FunctionCommon.mergeJson; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; -import com.google.common.collect.Lists; import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.function.Consumer; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; -import org.apache.distributedlog.api.namespace.Namespace; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.core.config.Configurator; import org.apache.pulsar.broker.authentication.AuthenticationParameters; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.common.functions.FunctionConfig; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.util.RestException; -import org.apache.pulsar.functions.api.Context; -import org.apache.pulsar.functions.api.Function; -import org.apache.pulsar.functions.instance.InstanceUtils; -import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.proto.Function.FunctionMetaData; -import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; -import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.proto.Function.SinkSpec; -import org.apache.pulsar.functions.proto.Function.SourceSpec; -import org.apache.pulsar.functions.proto.Function.SubscriptionType; -import org.apache.pulsar.functions.runtime.RuntimeFactory; -import org.apache.pulsar.functions.source.TopicSchema; -import org.apache.pulsar.functions.utils.FunctionCommon; +import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.utils.FunctionConfigUtils; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.LeaderService; -import org.apache.pulsar.functions.worker.PulsarWorkerService; -import org.apache.pulsar.functions.worker.WorkerConfig; -import org.apache.pulsar.functions.worker.WorkerUtils; -import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl; import org.apache.pulsar.functions.worker.rest.api.FunctionsImplV2; -import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; +import org.apache.pulsar.functions.worker.rest.api.v3.AbstractFunctionApiResourceTest; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -/** - * Unit test of {@link FunctionsApiV2Resource}. - */ -public class FunctionApiV2ResourceTest { - - private static final class TestFunction implements Function { - - @Override - public String process(String input, Context context) { - return input; - } - } - - private static final String tenant = "test-tenant"; - private static final String namespace = "test-namespace"; - private static final String function = "test-function"; - private static final String outputTopic = "test-output-topic"; - private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE; - private static final String className = TestFunction.class.getName(); - private SubscriptionType subscriptionType = SubscriptionType.FAILOVER; - private static final Map topicsToSerDeClassName = new HashMap<>(); - static { - topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE); - } - private static final int parallelism = 1; - - private PulsarWorkerService mockedWorkerService; - private PulsarAdmin mockedPulsarAdmin; - private Tenants mockedTenants; - private Namespaces mockedNamespaces; - private Functions mockedFunctions; - private TenantInfoImpl mockedTenantInfo; - private List namespaceList = new LinkedList<>(); - private FunctionMetaDataManager mockedManager; - private FunctionRuntimeManager mockedFunctionRunTimeManager; - private RuntimeFactory mockedRuntimeFactory; - private Namespace mockedNamespace; +public class FunctionApiV2ResourceTest extends AbstractFunctionApiResourceTest { private FunctionsImplV2 resource; - private InputStream mockedInputStream; - private FormDataContentDisposition mockedFormData; - private FunctionMetaData mockedFunctionMetadata; - private LeaderService mockedLeaderService; - private PulsarFunctionTestTemporaryDirectory tempDirectory; - private static Map mockStaticContexts = new HashMap<>(); - - @BeforeMethod - public void setup() throws Exception { - this.mockedManager = mock(FunctionMetaDataManager.class); - this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); - this.mockedTenantInfo = mock(TenantInfoImpl.class); - this.mockedRuntimeFactory = mock(RuntimeFactory.class); - this.mockedInputStream = mock(InputStream.class); - this.mockedNamespace = mock(Namespace.class); - this.mockedFormData = mock(FormDataContentDisposition.class); - when(mockedFormData.getFileName()).thenReturn("test"); - this.mockedPulsarAdmin = mock(PulsarAdmin.class); - this.mockedTenants = mock(Tenants.class); - this.mockedNamespaces = mock(Namespaces.class); - this.mockedFunctions = mock(Functions.class); - this.mockedLeaderService = mock(LeaderService.class); - this.mockedFunctionMetadata = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - namespaceList.add(tenant + "/" + namespace); - - this.mockedWorkerService = mock(PulsarWorkerService.class); - when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); - when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); - when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); - when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); - when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); - when(mockedWorkerService.isInitialized()).thenReturn(true); - when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); - when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); - when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); - when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); - when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); - when(mockedLeaderService.isLeader()).thenReturn(true); - when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata); - - // worker config - WorkerConfig workerConfig = new WorkerConfig() - .setWorkerId("test") - .setWorkerPort(8080) - .setFunctionMetadataTopicName("pulsar/functions") - .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); - tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); - tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); - when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); - - FunctionsImpl functions = spy(new FunctionsImpl(() -> mockedWorkerService)); - - this.resource = spy(new FunctionsImplV2(functions)); - - } - - @AfterMethod(alwaysRun = true) - public void cleanup() { - if (tempDirectory != null) { - tempDirectory.delete(); - } - mockStaticContexts.values().forEach(MockedStatic::close); - mockStaticContexts.clear(); - } - - private void mockStatic(Class classStatic, Consumer> consumer) { - final MockedStatic mockedStatic = - mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); - consumer.accept(mockedStatic); - } - - private void mockWorkerUtils() { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - }); - } - - private void mockWorkerUtils(Consumer> consumer) { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - if (consumer != null) { - consumer.accept(ctx); - } - }); - } - - private void mockInstanceUtils() { - mockStatic(InstanceUtils.class, ctx -> { - ctx.when(() -> InstanceUtils.calculateSubjectType(any())) - .thenReturn(FunctionDetails.ComponentType.FUNCTION); - }); - } - - // - // Register Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testRegisterFunctionMissingTenant() { - try { - testRegisterFunctionMissingArguments( - null, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testRegisterFunctionMissingNamespace() { - try { - testRegisterFunctionMissingArguments( - tenant, - null, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testRegisterFunctionMissingFunctionName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - null, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not provided") - public void testRegisterFunctionMissingPackage() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) specified for the function") - public void testRegisterFunctionMissingInputTopics() throws Exception { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not provided") - public void testRegisterFunctionMissingPackageDetails() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - null, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package does not have" - + " the correct format. Pulsar cannot determine if the package is a NAR package or JAR package. Function " - + "classname is not provided and attempts to load it as a NAR package produced the following error.*") - public void testRegisterFunctionMissingClassName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass must be in class path") - public void testRegisterFunctionWrongClassName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - "UnknownClass", - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a positive number") - public void testRegisterFunctionWrongParallelism() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - -2, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, - expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used as an input topic \\(topics must be one or the other\\)") - public void testRegisterFunctionSameInputOutput() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - topicsToSerDeClassName.keySet().iterator().next(), - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function + - "-output-topic/test:" + " is invalid") - public void testRegisterFunctionWrongOutputTopic() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - function + "-output-topic/test:", - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when getting Function package from .*") - public void testRegisterFunctionHttpUrl() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - null, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "http://localhost:1234/test"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testRegisterFunctionMissingArguments( - String tenant, - String namespace, - String function, - InputStream inputStream, - Map topicsToSerDeClassName, - FormDataContentDisposition details, - String outputTopic, - String outputSerdeClassName, - String className, - Integer parallelism, - String functionPkgUrl) { - FunctionConfig functionConfig = new FunctionConfig(); - if (tenant != null) { - functionConfig.setTenant(tenant); - } - if (namespace != null) { - functionConfig.setNamespace(namespace); - } - if (function != null) { - functionConfig.setName(function); - } - if (topicsToSerDeClassName != null) { - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - } - if (outputTopic != null) { - functionConfig.setOutput(outputTopic); - } - if (outputSerdeClassName != null) { - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - } - if (className != null) { - functionConfig.setClassName(className); - } - if (parallelism != null) { - functionConfig.setParallelism(parallelism); - } - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - - mockWorkerUtils(ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenCallRealMethod(); - }); - - try { - resource.registerFunction( - tenant, - namespace, - function, - inputStream, - details, - functionPkgUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - } - - private void registerDefaultFunction() { - FunctionConfig functionConfig = createDefaultFunctionConfig(); - try { - resource.registerFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - null, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists") - public void testRegisterExistedFunction() { - try { - Configurator.setRootLevel(Level.DEBUG); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") - public void testRegisterFunctionUploadFailure() throws Exception { - try { - mockWorkerUtils(ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenThrow(new IOException("upload failure")); - }); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - @Test - public void testRegisterFunctionSuccess() throws Exception { - try { - try (MockedStatic mocked = Mockito.mockStatic(WorkerUtils.class)) { - mocked.when(() -> WorkerUtils.uploadToBookKeeper( - any(Namespace.class), - any(InputStream.class), - anyString())).thenAnswer((i) -> null); - mocked.when(() -> WorkerUtils.dumpToTmpFile(any())).thenAnswer(i -> - { - try { - File tmpFile = FunctionCommon.createPkgTempFile(); - tmpFile.deleteOnExit(); - Files.copy((InputStream) i.getArguments()[0], tmpFile.toPath(), REPLACE_EXISTING); - return tmpFile; - } catch (IOException e) { - throw new RuntimeException("Cannot create a temporary file", e); - } - - } - ); - WorkerUtils.uploadToBookKeeper(null, null, null); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); - } - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist") - public void testRegisterFunctionNonExistingNamespace() { - try { - this.namespaceList.clear(); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant does not exist") - public void testRegisterFunctionNonexistantTenant() throws Exception { - try { - when(mockedTenants.getTenantInfo(any())).thenThrow(PulsarAdminException.NotFoundException.class); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") - public void testRegisterFunctionFailure() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - doThrow(new IllegalArgumentException("function failed to register")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted") - public void testRegisterFunctionInterrupted() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - doThrow(new IllegalStateException("Function registration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - // - // Update Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testUpdateFunctionMissingTenant() throws Exception { - try { - testUpdateFunctionMissingArguments( - null, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Tenant is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testUpdateFunctionMissingNamespace() throws Exception { - try { - testUpdateFunctionMissingArguments( - tenant, - null, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Namespace is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testUpdateFunctionMissingFunctionName() throws Exception { - try { - testUpdateFunctionMissingArguments( - tenant, - namespace, - null, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Function name is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingPackage() throws Exception { - try { - mockWorkerUtils(); - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingInputTopic() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingClassName() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test - public void testUpdateFunctionChangedParallelism() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism + 1, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + @Override + protected void doSetup() { + super.doSetup(); + this.resource = spy(new FunctionsImplV2(() -> mockedWorkerService)); } - @Test - public void testUpdateFunctionChangedInputs() throws Exception { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( + protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream, + FormDataContentDisposition details, String functionPkgUrl, + FunctionConfig functionConfig) throws IOException { + resource.registerFunction( tenant, namespace, function, - null, - topicsToSerDeClassName, - mockedFormData, - "DifferentOutput", - outputSerdeClassName, - null, - parallelism, + inputStream, + details, + functionPkgUrl, + JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig)), null); } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") - public void testUpdateFunctionChangedOutput() throws Exception { - try { - mockWorkerUtils(); - - Map someOtherInput = new HashMap<>(); - someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE); - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - someOtherInput, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - "Input Topics cannot be altered"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testUpdateFunctionMissingArguments( - String tenant, - String namespace, - String function, - InputStream inputStream, - Map topicsToSerDeClassName, - FormDataContentDisposition details, - String outputTopic, - String outputSerdeClassName, - String className, - Integer parallelism, - String expectedError) throws Exception { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - FunctionConfig functionConfig = new FunctionConfig(); - if (tenant != null) { - functionConfig.setTenant(tenant); - } - if (namespace != null) { - functionConfig.setNamespace(namespace); - } - if (function != null) { - functionConfig.setName(function); - } - if (topicsToSerDeClassName != null) { - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - } - if (outputTopic != null) { - functionConfig.setOutput(outputTopic); - } - if (outputSerdeClassName != null) { - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - } - if (className != null) { - functionConfig.setClassName(className); - } - if (parallelism != null) { - functionConfig.setParallelism(parallelism); - } - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - - if (expectedError != null) { - doThrow(new IllegalArgumentException(expectedError)) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - } - - try { - resource.updateFunction( - tenant, - namespace, - function, - inputStream, - details, - null, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - } - - private void updateDefaultFunction() { - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - - try { - resource.updateFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - null, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testUpdateNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") - public void testUpdateFunctionUploadFailure() throws Exception { - try { - mockWorkerUtils(ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenThrow(new IOException("upload failure")); - }); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - @Test - public void testUpdateFunctionSuccess() throws Exception { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - updateDefaultFunction(); - } - - @Test - public void testUpdateFunctionWithUrl() throws Exception { - Configurator.setRootLevel(Level.DEBUG); - - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; - - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - try { - resource.updateFunction( - tenant, - namespace, - function, - null, - null, - filePackageUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") - public void testUpdateFunctionFailure() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalArgumentException("function failed to register")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted") - public void testUpdateFunctionInterrupted() throws Exception { - try { - mockWorkerUtils(); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalStateException("Function registeration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - // - // deregister function - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testDeregisterFunctionMissingTenant() { - try { - - testDeregisterFunctionMissingArguments( - null, - namespace, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + protected void updateFunction(String tenant, + String namespace, + String functionName, + InputStream uploadedInputStream, + FormDataContentDisposition fileDetail, + String functionPkgUrl, + FunctionConfig functionConfig, + AuthenticationParameters authParams, + UpdateOptionsImpl updateOptions) throws IOException { + resource.updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, functionPkgUrl, + JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig)), authParams); } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testDeregisterFunctionMissingNamespace() { - try { - testDeregisterFunctionMissingArguments( - tenant, - null, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; + protected File downloadFunction(final String path, final AuthenticationParameters authParams) + throws IOException { + Response response = resource.downloadFunction(path, authParams); + StreamingOutput streamingOutput = readEntity(response, StreamingOutput.class); + File pkgFile = File.createTempFile("testpkg", "nar"); + try (OutputStream output = new FileOutputStream(pkgFile)) { + streamingOutput.write(output); } + return pkgFile; } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testDeregisterFunctionMissingFunctionName() { - try { - testDeregisterFunctionMissingArguments( - tenant, - namespace, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + private T readEntity(Response response, Class clazz) { + return clazz.cast(response.getEntity()); } - private void testDeregisterFunctionMissingArguments( + protected void testDeregisterFunctionMissingArguments( String tenant, String namespace, String function @@ -1145,112 +102,18 @@ private void testDeregisterFunctionMissingArguments( tenant, namespace, function, - AuthenticationParameters.builder().build()); + null); } - private void deregisterDefaultFunction() { + protected void deregisterDefaultFunction() { resource.deregisterFunction( tenant, namespace, function, - AuthenticationParameters.builder().build()); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testDeregisterNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); - throw re; - } - } - - @Test - public void testDeregisterFunctionSuccess() { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - deregisterDefaultFunction(); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to deregister") - public void testDeregisterFunctionFailure() throws Exception { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalArgumentException("function failed to deregister")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration interrupted") - public void testDeregisterFunctionInterrupted() throws Exception { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalStateException("Function deregisteration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), anyBoolean()); - - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - // - // Get Function Info - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testGetFunctionMissingTenant() throws IOException { - try { - testGetFunctionMissingArguments( - null, - namespace, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testGetFunctionMissingNamespace() throws IOException { - try { - testGetFunctionMissingArguments( - tenant, - null, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testGetFunctionMissingFunctionName() throws IOException { - try { - testGetFunctionMissingArguments( - tenant, - namespace, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + null); } - private void testGetFunctionMissingArguments( + protected void testGetFunctionMissingArguments( String tenant, String namespace, String function @@ -1258,20 +121,36 @@ private void testGetFunctionMissingArguments( resource.getFunctionInfo( tenant, namespace, - function, - AuthenticationParameters.builder().build() + function, null + ); + } + + protected void testListFunctionsMissingArguments( + String tenant, + String namespace + ) { + resource.listFunctions( + tenant, + namespace, null ); } - private FunctionDetails getDefaultFunctionInfo() throws IOException { + protected List listDefaultFunctions() { + return new Gson().fromJson(readEntity(resource.listFunctions( + tenant, + namespace, null + ), String.class), List.class); + } + + private Function.FunctionDetails getDefaultFunctionInfo() throws IOException { String json = (String) resource.getFunctionInfo( tenant, namespace, function, AuthenticationParameters.builder().build() ).getEntity(); - FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); + Function.FunctionDetails.Builder functionDetailsBuilder = Function.FunctionDetails.newBuilder(); mergeJson(json, functionDetailsBuilder); return functionDetailsBuilder.build(); } @@ -1292,218 +171,31 @@ public void testGetFunctionSuccess() throws IOException { mockInstanceUtils(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - SinkSpec sinkSpec = SinkSpec.newBuilder() + Function.SinkSpec sinkSpec = Function.SinkSpec.newBuilder() .setTopic(outputTopic) .setSerDeClassName(outputSerdeClassName).build(); - FunctionDetails functionDetails = FunctionDetails.newBuilder() + Function.FunctionDetails functionDetails = Function.FunctionDetails.newBuilder() .setClassName(className) .setSink(sinkSpec) .setName(function) .setNamespace(namespace) - .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE) + .setProcessingGuarantees(Function.ProcessingGuarantees.ATMOST_ONCE) .setAutoAck(true) .setTenant(tenant) .setParallelism(parallelism) - .setSource(SourceSpec.newBuilder().setSubscriptionType(subscriptionType) + .setSource(Function.SourceSpec.newBuilder().setSubscriptionType(subscriptionType) .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build(); - FunctionMetaData metaData = FunctionMetaData.newBuilder() + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() .setCreateTime(System.currentTimeMillis()) .setFunctionDetails(functionDetails) - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) .setVersion(1234) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - FunctionDetails actual = getDefaultFunctionInfo(); + Function.FunctionDetails actual = getDefaultFunctionInfo(); assertEquals( functionDetails, actual); } - - // - // List Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testListFunctionsMissingTenant() { - try { - testListFunctionsMissingArguments( - null, - namespace - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testListFunctionsMissingNamespace() { - try { - testListFunctionsMissingArguments( - tenant, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testListFunctionsMissingArguments( - String tenant, - String namespace - ) { - resource.listFunctions( - tenant, - namespace, - AuthenticationParameters.builder().build() - ); - - } - - private List listDefaultFunctions() { - return new Gson().fromJson((String) resource.listFunctions( - tenant, - namespace, - AuthenticationParameters.builder().build() - ).getEntity(), List.class); - } - - @Test - public void testListFunctionsSuccess() { - mockInstanceUtils(); - final List functions = Lists.newArrayList("test-1", "test-2"); - final List metaDataList = new LinkedList<>(); - FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-1").build() - ).build(); - FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-2").build() - ).build(); - metaDataList.add(functionMetaData1); - metaDataList.add(functionMetaData2); - when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList); - - List functionList = listDefaultFunctions(); - assertEquals(functions, functionList); - } - - @Test - public void testDownloadFunctionHttpUrl() throws Exception { - String jarHttpUrl = - "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; - String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService); - StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction(jarHttpUrl, - AuthenticationParameters.builder().build()).getEntity(); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - Assert.assertTrue(pkgFile.exists()); - if (pkgFile.exists()) { - pkgFile.delete(); - } - } - - @Test - public void testDownloadFunctionFile() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String testDir = FunctionApiV2ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - FunctionsImplV2 function = new FunctionsImplV2(() -> mockedWorkerService); - StreamingOutput streamOutput = (StreamingOutput) function.downloadFunction("file:///" + fileLocation, - AuthenticationParameters.builder().build()).getEntity(); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - Assert.assertTrue(pkgFile.exists()); - if (pkgFile.exists()) { - pkgFile.delete(); - } - } - - @Test - public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { - Configurator.setRootLevel(Level.DEBUG); - - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - try { - resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - } - - @Test - public void testRegisterFunctionWithConflictingFields() throws Exception { - Configurator.setRootLevel(Level.DEBUG); - String actualTenant = "DIFFERENT_TENANT"; - String actualNamespace = "DIFFERENT_NAMESPACE"; - String actualName = "DIFFERENT_NAME"; - this.namespaceList.add(actualTenant + "/" + actualNamespace); - - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); - - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - try { - resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, - JsonFormat.printer().print(FunctionConfigUtils.convert(functionConfig, (ClassLoader) null)), - AuthenticationParameters.builder().build()); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - } - - public static FunctionConfig createDefaultFunctionConfig() { - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - return functionConfig; - } - - public static FunctionDetails createDefaultFunctionDetails() { - FunctionConfig functionConfig = createDefaultFunctionConfig(); - return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); - } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java new file mode 100644 index 0000000000000..5845ff3afd9ac --- /dev/null +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java @@ -0,0 +1,1367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.pulsar.functions.worker.rest.api.v3; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import com.google.common.collect.Lists; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import javax.ws.rs.core.Response; +import org.apache.distributedlog.api.namespace.Namespace; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.functions.FunctionConfig; +import org.apache.pulsar.common.functions.UpdateOptionsImpl; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.RestException; +import org.apache.pulsar.functions.api.Context; +import org.apache.pulsar.functions.api.Function; +import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.FunctionMetaData; +import org.apache.pulsar.functions.proto.Function.SubscriptionType; +import org.apache.pulsar.functions.source.TopicSchema; +import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.apache.pulsar.functions.worker.WorkerUtils; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +public abstract class AbstractFunctionApiResourceTest extends AbstractFunctionsResourceTest { + + @Test + public void testListFunctionsSuccess() { + mockInstanceUtils(); + final List functions = Lists.newArrayList("test-1", "test-2"); + final List metaDataList = new LinkedList<>(); + FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder().setName("test-1").build() + ).build(); + FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder().setName("test-2").build() + ).build(); + metaDataList.add(functionMetaData1); + metaDataList.add(functionMetaData2); + when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList); + + List functionList = listDefaultFunctions(); + assertEquals(functions, functionList); + } + + @Test + public void testOnlyGetSources() { + List functions = Lists.newArrayList("test-2"); + List functionMetaDataList = new LinkedList<>(); + FunctionMetaData f1 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder() + .setName("test-1") + .setComponentType(FunctionDetails.ComponentType.SOURCE) + .build()).build(); + functionMetaDataList.add(f1); + FunctionMetaData f2 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder() + .setName("test-2") + .setComponentType(FunctionDetails.ComponentType.FUNCTION) + .build()).build(); + functionMetaDataList.add(f2); + FunctionMetaData f3 = FunctionMetaData.newBuilder().setFunctionDetails( + FunctionDetails.newBuilder() + .setName("test-3") + .setComponentType(FunctionDetails.ComponentType.SINK) + .build()).build(); + functionMetaDataList.add(f3); + when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList); + + List functionList = listDefaultFunctions(); + assertEquals(functions, functionList); + } + + private static final class TestFunction implements Function { + + @Override + public String process(String input, Context context) { + return input; + } + } + + private static final class WrongFunction implements Consumer { + @Override + public void accept(String s) { + + } + } + + protected static final String function = "test-function"; + protected static final String outputTopic = "test-output-topic"; + protected static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE; + protected static final String className = TestFunction.class.getName(); + protected SubscriptionType subscriptionType = SubscriptionType.FAILOVER; + protected FunctionMetaData mockedFunctionMetadata; + + + @Override + protected void doSetup() { + this.mockedFunctionMetadata = + FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata); + } + + @Override + protected FunctionDetails.ComponentType getComponentType() { + return FunctionDetails.ComponentType.FUNCTION; + } + + + abstract protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream, + FormDataContentDisposition details, String functionPkgUrl, FunctionConfig functionConfig) + throws IOException; + abstract protected void updateFunction(String tenant, + String namespace, + String functionName, + InputStream uploadedInputStream, + FormDataContentDisposition fileDetail, + String functionPkgUrl, + FunctionConfig functionConfig, + AuthenticationParameters authParams, + UpdateOptionsImpl updateOptions) throws IOException; + + abstract protected File downloadFunction(final String path, final AuthenticationParameters authParams) + throws IOException; + + abstract protected void testDeregisterFunctionMissingArguments( + String tenant, + String namespace, + String function + ); + + abstract protected void deregisterDefaultFunction(); + + abstract protected void testGetFunctionMissingArguments( + String tenant, + String namespace, + String function + ) throws IOException; + + abstract protected void testListFunctionsMissingArguments( + String tenant, + String namespace + ); + + abstract protected List listDefaultFunctions(); + + // + // Register Functions + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testRegisterFunctionMissingTenant() throws IOException { + try { + testRegisterFunctionMissingArguments( + null, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testRegisterFunctionMissingNamespace() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + null, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") + public void testRegisterFunctionMissingFunctionName() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + null, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not " + + "provided") + public void testRegisterFunctionMissingPackage() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) " + + "specified for the function") + public void testRegisterFunctionMissingInputTopics() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + null, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not " + + "provided") + public void testRegisterFunctionMissingPackageDetails() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + null, + outputTopic, + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, + expectedExceptionsMessageRegExp = "Function class name is not provided.") + public void testRegisterFunctionMissingClassName() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + null, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass " + + "must be in class path") + public void testRegisterFunctionWrongClassName() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + "UnknownClass", + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a" + + " positive number") + public void testRegisterFunctionWrongParallelism() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + -2, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, + expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used " + + "as an input topic \\(topics must be one or the other\\)") + public void testRegisterFunctionSameInputOutput() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + topicsToSerDeClassName.keySet().iterator().next(), + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function + + "-output-topic/test:" + " is invalid") + public void testRegisterFunctionWrongOutputTopic() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + function + "-output-topic/test:", + outputSerdeClassName, + className, + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when " + + "getting Function package from .*") + public void testRegisterFunctionHttpUrl() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + null, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "http://localhost:1234/test"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class .*. does not " + + "implement the correct interface") + public void testRegisterFunctionImplementWrongInterface() throws IOException { + try { + testRegisterFunctionMissingArguments( + tenant, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + WrongFunction.class.getName(), + parallelism, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + private void testRegisterFunctionMissingArguments( + String tenant, + String namespace, + String function, + InputStream inputStream, + Map topicsToSerDeClassName, + FormDataContentDisposition details, + String outputTopic, + String outputSerdeClassName, + String className, + Integer parallelism, + String functionPkgUrl) throws IOException { + FunctionConfig functionConfig = new FunctionConfig(); + if (tenant != null) { + functionConfig.setTenant(tenant); + } + if (namespace != null) { + functionConfig.setNamespace(namespace); + } + if (function != null) { + functionConfig.setName(function); + } + if (topicsToSerDeClassName != null) { + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + } + if (outputTopic != null) { + functionConfig.setOutput(outputTopic); + } + if (outputSerdeClassName != null) { + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + } + if (className != null) { + functionConfig.setClassName(className); + } + if (parallelism != null) { + functionConfig.setParallelism(parallelism); + } + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + + registerFunction(tenant, namespace, function, inputStream, details, functionPkgUrl, functionConfig); + + } + + @Test(expectedExceptions = Exception.class, expectedExceptionsMessageRegExp = "Function config is not provided") + public void testUpdateMissingFunctionConfig() throws IOException { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + updateFunction( + tenant, + namespace, + function, + mockedInputStream, + mockedFormData, + null, + null, + null, null); + } + + + private void registerDefaultFunction() throws IOException { + registerDefaultFunctionWithPackageUrl(null); + } + + private void registerDefaultFunctionWithPackageUrl(String packageUrl) throws IOException { + FunctionConfig functionConfig = createDefaultFunctionConfig(); + registerFunction(tenant, namespace, function, mockedInputStream, mockedFormData, packageUrl, functionConfig); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already" + + " exists") + public void testRegisterExistedFunction() throws IOException { + try { + Configurator.setRootLevel(Level.DEBUG); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") + public void testRegisterFunctionUploadFailure() throws IOException { + try { + mockWorkerUtils(ctx -> { + ctx.when(() -> { + WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class)); + } + ).thenThrow(new IOException("upload failure")); + }); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + @Test + public void testRegisterFunctionSuccess() throws IOException { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(timeOut = 20000) + public void testRegisterFunctionSuccessWithPackageName() throws IOException { + registerDefaultFunctionWithPackageUrl("function://public/default/test@v1"); + } + + @Test(timeOut = 20000) + public void testRegisterFunctionFailedWithWrongPackageName() throws PulsarAdminException, IOException { + try { + doThrow(new PulsarAdminException("package name is invalid")) + .when(mockedPackages).download(anyString(), anyString()); + registerDefaultFunctionWithPackageUrl("function://"); + } catch (RestException e) { + // expected exception + assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist") + public void testRegisterFunctionNonExistingNamespace() throws IOException { + try { + this.namespaceList.clear(); + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant does not exist") + public void testRegisterFunctionNonexistantTenant() throws Exception { + try { + when(mockedTenants.getTenantInfo(any())).thenThrow(PulsarAdminException.NotFoundException.class); + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") + public void testRegisterFunctionFailure() throws Exception { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + doThrow(new IllegalArgumentException("function failed to register")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration " + + "interrupted") + public void testRegisterFunctionInterrupted() throws Exception { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + doThrow(new IllegalStateException("Function registration interrupted")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + registerDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + // + // Update Functions + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testUpdateFunctionMissingTenant() throws Exception { + try { + testUpdateFunctionMissingArguments( + null, + namespace, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Tenant is not provided"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testUpdateFunctionMissingNamespace() throws Exception { + try { + testUpdateFunctionMissingArguments( + tenant, + null, + function, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Namespace is not provided"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") + public void testUpdateFunctionMissingFunctionName() throws Exception { + try { + testUpdateFunctionMissingArguments( + tenant, + namespace, + null, + mockedInputStream, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Function name is not provided"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") + public void testUpdateFunctionMissingPackage() throws Exception { + try { + mockWorkerUtils(); + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Update contains no change"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") + public void testUpdateFunctionMissingInputTopic() throws Exception { + try { + mockWorkerUtils(); + + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + null, + mockedFormData, + outputTopic, + outputSerdeClassName, + className, + parallelism, + "Update contains no change"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") + public void testUpdateFunctionMissingClassName() throws Exception { + try { + mockWorkerUtils(); + + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + null, + parallelism, + "Update contains no change"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test + public void testUpdateFunctionChangedParallelism() throws Exception { + try { + mockWorkerUtils(); + + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + outputTopic, + outputSerdeClassName, + null, + parallelism + 1, + null); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test + public void testUpdateFunctionChangedInputs() throws Exception { + mockWorkerUtils(); + + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + topicsToSerDeClassName, + mockedFormData, + "DifferentOutput", + outputSerdeClassName, + null, + parallelism, + null); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") + public void testUpdateFunctionChangedOutput() throws Exception { + try { + mockWorkerUtils(); + + Map someOtherInput = new HashMap<>(); + someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE); + testUpdateFunctionMissingArguments( + tenant, + namespace, + function, + null, + someOtherInput, + mockedFormData, + outputTopic, + outputSerdeClassName, + null, + parallelism, + "Input Topics cannot be altered"); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + private void testUpdateFunctionMissingArguments( + String tenant, + String namespace, + String function, + InputStream inputStream, + Map topicsToSerDeClassName, + FormDataContentDisposition details, + String outputTopic, + String outputSerdeClassName, + String className, + Integer parallelism, + String expectedError) throws Exception { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + FunctionConfig functionConfig = new FunctionConfig(); + if (tenant != null) { + functionConfig.setTenant(tenant); + } + if (namespace != null) { + functionConfig.setNamespace(namespace); + } + if (function != null) { + functionConfig.setName(function); + } + if (topicsToSerDeClassName != null) { + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + } + if (outputTopic != null) { + functionConfig.setOutput(outputTopic); + } + if (outputSerdeClassName != null) { + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + } + if (className != null) { + functionConfig.setClassName(className); + } + if (parallelism != null) { + functionConfig.setParallelism(parallelism); + } + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + + if (expectedError != null) { + doThrow(new IllegalArgumentException(expectedError)) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + } + + updateFunction( + tenant, + namespace, + function, + inputStream, + details, + null, + functionConfig, + null, null); + + } + + private void updateDefaultFunction() throws IOException { + updateDefaultFunctionWithPackageUrl(null); + } + + private void updateDefaultFunctionWithPackageUrl(String packageUrl) throws IOException { + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + + updateFunction( + tenant, + namespace, + function, + mockedInputStream, + mockedFormData, + packageUrl, + functionConfig, + null, null); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't" + + " exist") + public void testUpdateNotExistedFunction() throws IOException { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + updateDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") + public void testUpdateFunctionUploadFailure() throws Exception { + try { + mockWorkerUtils(ctx -> { + ctx.when(() -> { + WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class)); + + }).thenThrow(new IOException("upload failure")); + }); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + updateDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + @Test + public void testUpdateFunctionSuccess() throws Exception { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + updateDefaultFunction(); + } + + @Test + public void testUpdateFunctionWithUrl() throws IOException { + Configurator.setRootLevel(Level.DEBUG); + + String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + String filePackageUrl = "file://" + fileLocation; + + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + updateFunction( + tenant, + namespace, + function, + null, + null, + filePackageUrl, + functionConfig, + null, null); + + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") + public void testUpdateFunctionFailure() throws Exception { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + doThrow(new IllegalArgumentException("function failed to register")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + updateDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration " + + "interrupted") + public void testUpdateFunctionInterrupted() throws Exception { + try { + mockWorkerUtils(); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + doThrow(new IllegalStateException("Function registeration interrupted")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + updateDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + + @Test(timeOut = 20000) + public void testUpdateFunctionSuccessWithPackageName() throws IOException { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + updateDefaultFunctionWithPackageUrl("function://public/default/test@v1"); + } + + @Test(timeOut = 20000) + public void testUpdateFunctionFailedWithWrongPackageName() throws PulsarAdminException, IOException { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + try { + doThrow(new PulsarAdminException("package name is invalid")) + .when(mockedPackages).download(anyString(), anyString()); + registerDefaultFunctionWithPackageUrl("function://"); + } catch (RestException e) { + // expected exception + assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + } + } + + // + // deregister function + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testDeregisterFunctionMissingTenant() { + try { + + testDeregisterFunctionMissingArguments( + null, + namespace, + function + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testDeregisterFunctionMissingNamespace() { + try { + testDeregisterFunctionMissingArguments( + tenant, + null, + function + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") + public void testDeregisterFunctionMissingFunctionName() { + try { + testDeregisterFunctionMissingArguments( + tenant, + namespace, + null + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't" + + " exist") + public void testDeregisterNotExistedFunction() { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + deregisterDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); + throw re; + } + } + + @Test + public void testDeregisterFunctionSuccess() { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + deregisterDefaultFunction(); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to deregister") + public void testDeregisterFunctionFailure() throws Exception { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + doThrow(new IllegalArgumentException("function failed to deregister")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + deregisterDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration " + + "interrupted") + public void testDeregisterFunctionInterrupted() throws Exception { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + doThrow(new IllegalStateException("Function deregisteration interrupted")) + .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); + + deregisterDefaultFunction(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); + throw re; + } + } + + // + // Get Function Info + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testGetFunctionMissingTenant() throws IOException { + try { + testGetFunctionMissingArguments( + null, + namespace, + function + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testGetFunctionMissingNamespace() throws IOException { + try { + testGetFunctionMissingArguments( + tenant, + null, + function + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") + public void testGetFunctionMissingFunctionName() throws IOException { + try { + testGetFunctionMissingArguments( + tenant, + namespace, + null + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + // + // List Functions + // + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") + public void testListFunctionsMissingTenant() { + try { + testListFunctionsMissingArguments( + null, + namespace + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") + public void testListFunctionsMissingNamespace() { + try { + testListFunctionsMissingArguments( + tenant, + null + ); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); + throw re; + } + } + + @Test + public void testDownloadFunctionHttpUrl() throws Exception { + String jarHttpUrl = + "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; + File pkgFile = downloadFunction(jarHttpUrl, null); + pkgFile.delete(); + } + + @Test + public void testDownloadFunctionFile() throws Exception { + File file = getPulsarApiExamplesNar(); + File pkgFile = downloadFunction(file.toURI().toString(), null); + Assert.assertTrue(pkgFile.exists()); + Assert.assertEquals(file.length(), pkgFile.length()); + pkgFile.delete(); + } + + @Test + public void testDownloadFunctionBuiltinConnector() throws Exception { + File file = getPulsarApiExamplesNar(); + + WorkerConfig config = new WorkerConfig() + .setUploadBuiltinSinksSources(false); + when(mockedWorkerService.getWorkerConfig()).thenReturn(config); + + registerBuiltinConnector("cassandra", file); + + File pkgFile = downloadFunction("builtin://cassandra", null); + Assert.assertTrue(pkgFile.exists()); + Assert.assertEquals(file.length(), pkgFile.length()); + pkgFile.delete(); + } + + @Test + public void testDownloadFunctionBuiltinFunction() throws Exception { + File file = getPulsarApiExamplesNar(); + + WorkerConfig config = new WorkerConfig() + .setUploadBuiltinSinksSources(false); + when(mockedWorkerService.getWorkerConfig()).thenReturn(config); + + registerBuiltinFunction("exclamation", file); + + File pkgFile = downloadFunction("builtin://exclamation", null); + Assert.assertTrue(pkgFile.exists()); + Assert.assertEquals(file.length(), pkgFile.length()); + pkgFile.delete(); + } + + @Test + public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { + Configurator.setRootLevel(Level.DEBUG); + + File file = getPulsarApiExamplesNar(); + String filePackageUrl = file.toURI().toString(); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig); + + } + + @Test + public void testRegisterFunctionWithConflictingFields() throws Exception { + Configurator.setRootLevel(Level.DEBUG); + String actualTenant = "DIFFERENT_TENANT"; + String actualNamespace = "DIFFERENT_NAMESPACE"; + String actualName = "DIFFERENT_NAME"; + this.namespaceList.add(actualTenant + "/" + actualNamespace); + + File file = getPulsarApiExamplesNar(); + String filePackageUrl = file.toURI().toString(); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); + + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, functionConfig); + } + + public static FunctionConfig createDefaultFunctionConfig() { + FunctionConfig functionConfig = new FunctionConfig(); + functionConfig.setTenant(tenant); + functionConfig.setNamespace(namespace); + functionConfig.setName(function); + functionConfig.setClassName(className); + functionConfig.setParallelism(parallelism); + functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); + functionConfig.setOutput(outputTopic); + functionConfig.setOutputSerdeClassName(outputSerdeClassName); + functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); + return functionConfig; + } + + public static FunctionDetails createDefaultFunctionDetails() { + FunctionConfig functionConfig = createDefaultFunctionConfig(); + return FunctionConfigUtils.convert(functionConfig); + } +} diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java new file mode 100644 index 0000000000000..4cc4ed0b09819 --- /dev/null +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker.rest.api.v3; + +import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import org.apache.distributedlog.api.namespace.Namespace; +import org.apache.pulsar.client.admin.Functions; +import org.apache.pulsar.client.admin.Namespaces; +import org.apache.pulsar.client.admin.Packages; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.Tenants; +import org.apache.pulsar.common.functions.FunctionDefinition; +import org.apache.pulsar.common.io.ConnectorDefinition; +import org.apache.pulsar.common.nar.NarClassLoader; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.functions.instance.InstanceUtils; +import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.functions.runtime.RuntimeFactory; +import org.apache.pulsar.functions.source.TopicSchema; +import org.apache.pulsar.functions.utils.LoadedFunctionPackage; +import org.apache.pulsar.functions.utils.ValidatableFunctionPackage; +import org.apache.pulsar.functions.utils.functions.FunctionArchive; +import org.apache.pulsar.functions.utils.functions.FunctionUtils; +import org.apache.pulsar.functions.utils.io.Connector; +import org.apache.pulsar.functions.utils.io.ConnectorUtils; +import org.apache.pulsar.functions.worker.ConnectorsManager; +import org.apache.pulsar.functions.worker.FunctionMetaDataManager; +import org.apache.pulsar.functions.worker.FunctionRuntimeManager; +import org.apache.pulsar.functions.worker.FunctionsManager; +import org.apache.pulsar.functions.worker.LeaderService; +import org.apache.pulsar.functions.worker.PulsarWorkerService; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.apache.pulsar.functions.worker.WorkerUtils; +import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.mockito.Answers; +import org.mockito.MockSettings; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; + +public abstract class AbstractFunctionsResourceTest { + + protected static final String tenant = "test-tenant"; + protected static final String namespace = "test-namespace"; + protected static final Map topicsToSerDeClassName = new HashMap<>(); + protected static final String subscriptionName = "test-subscription"; + protected static final String CASSANDRA_STRING_SINK = "org.apache.pulsar.io.cassandra.CassandraStringSink"; + protected static final int parallelism = 1; + private static final String SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH = "pulsar-io-cassandra.nar.path"; + private static final String SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH = "pulsar-io-twitter.nar.path"; + private static final String SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH = "pulsar-io-invalid.nar.path"; + private static final String SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH = + "pulsar-functions-api-examples.nar.path"; + protected static Map mockStaticContexts = new HashMap<>(); + + static { + topicsToSerDeClassName.put("test_src", DEFAULT_SERDE); + topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE); + } + + protected PulsarWorkerService mockedWorkerService; + protected PulsarAdmin mockedPulsarAdmin; + protected Tenants mockedTenants; + protected Namespaces mockedNamespaces; + protected Functions mockedFunctions; + protected TenantInfoImpl mockedTenantInfo; + protected List namespaceList = new LinkedList<>(); + protected FunctionMetaDataManager mockedManager; + protected FunctionRuntimeManager mockedFunctionRunTimeManager; + protected RuntimeFactory mockedRuntimeFactory; + protected Namespace mockedNamespace; + protected InputStream mockedInputStream; + protected FormDataContentDisposition mockedFormData; + protected Function.FunctionMetaData mockedFunctionMetaData; + protected LeaderService mockedLeaderService; + protected Packages mockedPackages; + protected PulsarFunctionTestTemporaryDirectory tempDirectory; + protected ConnectorsManager connectorsManager = new ConnectorsManager(); + protected FunctionsManager functionsManager = new FunctionsManager(); + + public static File getPulsarIOCassandraNar() { + return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH) + , "pulsar-io-cassandra.nar file location must be specified with " + + SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH + " system property")); + } + + public static File getPulsarIOTwitterNar() { + return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH) + , "pulsar-io-twitter.nar file location must be specified with " + + SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH + " system property")); + } + + public static File getPulsarIOInvalidNar() { + return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH) + , "invalid nar file location must be specified with " + + SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH + " system property")); + } + + public static File getPulsarApiExamplesNar() { + return new File(Objects.requireNonNull( + System.getProperty(SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH) + , "pulsar-functions-api-examples.nar file location must be specified with " + + SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH + " system property")); + } + + @BeforeMethod + public final void setup() throws Exception { + this.mockedManager = mock(FunctionMetaDataManager.class); + this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); + this.mockedRuntimeFactory = mock(RuntimeFactory.class); + this.mockedInputStream = mock(InputStream.class); + this.mockedNamespace = mock(Namespace.class); + this.mockedFormData = mock(FormDataContentDisposition.class); + when(mockedFormData.getFileName()).thenReturn("test"); + this.mockedTenantInfo = mock(TenantInfoImpl.class); + this.mockedPulsarAdmin = mock(PulsarAdmin.class); + this.mockedTenants = mock(Tenants.class); + this.mockedNamespaces = mock(Namespaces.class); + this.mockedFunctions = mock(Functions.class); + this.mockedLeaderService = mock(LeaderService.class); + this.mockedPackages = mock(Packages.class); + namespaceList.add(tenant + "/" + namespace); + + this.mockedWorkerService = mock(PulsarWorkerService.class); + when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); + when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); + when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); + when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); + when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); + when(mockedWorkerService.isInitialized()).thenReturn(true); + when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); + when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); + when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); + when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); + when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); + when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages); + when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); + when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); + when(mockedLeaderService.isLeader()).thenReturn(true); + doAnswer(invocationOnMock -> { + Files.copy(getDefaultNarFile().toPath(), Paths.get(invocationOnMock.getArgument(1, String.class)), + StandardCopyOption.REPLACE_EXISTING); + return null; + }).when(mockedPackages).download(any(), any()); + + // worker config + WorkerConfig workerConfig = new WorkerConfig() + .setWorkerId("test") + .setWorkerPort(8080) + .setFunctionMetadataTopicName("pulsar/functions") + .setNumFunctionPackageReplicas(3) + .setPulsarServiceUrl("pulsar://localhost:6650/"); + tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); + tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); + when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); + when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); + when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); + + doSetup(); + } + + protected File getDefaultNarFile() { + return getPulsarIOTwitterNar(); + } + + protected void doSetup() throws Exception { + + } + + @AfterMethod(alwaysRun = true) + public void cleanup() { + if (tempDirectory != null) { + tempDirectory.delete(); + } + mockStaticContexts.values().forEach(MockedStatic::close); + mockStaticContexts.clear(); + } + + protected void mockStatic(Class classStatic, Consumer> consumer) { + mockStatic(classStatic, withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS), consumer); + } + + private void mockStatic(Class classStatic, MockSettings mockSettings, Consumer> consumer) { + final MockedStatic mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), + name -> Mockito.mockStatic(classStatic, mockSettings)); + consumer.accept(mockedStatic); + } + + protected void mockWorkerUtils() { + mockWorkerUtils(null); + } + + protected void mockWorkerUtils(Consumer> consumer) { + mockStatic(WorkerUtils.class, withSettings(), ctx -> { + // make dumpToTmpFile return the nar file copy + ctx.when(() -> WorkerUtils.dumpToTmpFile(mockedInputStream)) + .thenAnswer(invocation -> { + Path tempFile = Files.createTempFile("test", ".nar"); + Files.copy(getPulsarApiExamplesNar().toPath(), tempFile, + StandardCopyOption.REPLACE_EXISTING); + return tempFile.toFile(); + }); + ctx.when(() -> WorkerUtils.dumpToTmpFile(any())) + .thenAnswer(Answers.CALLS_REAL_METHODS); + if (consumer != null) { + consumer.accept(ctx); + } + }); + } + + protected void mockInstanceUtils() { + mockStatic(InstanceUtils.class, ctx -> { + ctx.when(() -> InstanceUtils.calculateSubjectType(any())) + .thenReturn(getComponentType()); + }); + } + + protected abstract Function.FunctionDetails.ComponentType getComponentType(); + + public static class LoadedConnector extends Connector { + public LoadedConnector(ConnectorDefinition connectorDefinition) { + super(null, connectorDefinition, null, true); + } + + @Override + public ValidatableFunctionPackage getConnectorFunctionPackage() { + return new LoadedFunctionPackage(getClass().getClassLoader(), ConnectorDefinition.class, + getConnectorDefinition()); + } + } + + + protected void registerBuiltinConnector(String connectorType, String className) { + ConnectorDefinition connectorDefinition = null; + if (className != null) { + connectorDefinition = new ConnectorDefinition(); + // set source and sink class to the same to simplify the test + connectorDefinition.setSinkClass(className); + connectorDefinition.setSourceClass(className); + } + connectorsManager.addConnector(connectorType, new LoadedConnector(connectorDefinition)); + } + + protected void registerBuiltinConnector(String connectorType, File packageFile) { + ConnectorDefinition cntDef; + try { + cntDef = ConnectorUtils.getConnectorDefinition(packageFile); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + connectorsManager.addConnector(connectorType, + new Connector(packageFile.toPath(), cntDef, NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, true)); + } + + public static class LoadedFunctionArchive extends FunctionArchive { + public LoadedFunctionArchive(FunctionDefinition functionDefinition) { + super(null, functionDefinition, null, true); + } + + @Override + public ValidatableFunctionPackage getFunctionPackage() { + return new LoadedFunctionPackage(getClass().getClassLoader(), FunctionDefinition.class, + getFunctionDefinition()); + } + } + + protected void registerBuiltinFunction(String functionType, String className) { + FunctionDefinition functionDefinition = null; + if (className != null) { + functionDefinition = new FunctionDefinition(); + functionDefinition.setFunctionClass(className); + } + functionsManager.addFunction(functionType, new LoadedFunctionArchive(functionDefinition)); + } + + protected void registerBuiltinFunction(String functionType, File packageFile) { + FunctionDefinition cntDef; + try { + cntDef = FunctionUtils.getFunctionDefinition(packageFile); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + functionsManager.addFunction(functionType, + new FunctionArchive(packageFile.toPath(), cntDef, NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR, true)); + } +} diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index 0c20083bb89ca..a1a418460be45 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -18,1791 +18,168 @@ */ package org.apache.pulsar.functions.worker.rest.api.v3; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; -import com.google.common.collect.Lists; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URL; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.function.Consumer; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.pulsar.broker.authentication.AuthenticationParameters; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.Packages; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.UpdateOptionsImpl; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.RestException; -import org.apache.pulsar.functions.api.Context; -import org.apache.pulsar.functions.api.Function; -import org.apache.pulsar.functions.instance.InstanceUtils; -import org.apache.pulsar.functions.proto.Function.FunctionDetails; -import org.apache.pulsar.functions.proto.Function.FunctionMetaData; -import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; -import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.proto.Function.SinkSpec; -import org.apache.pulsar.functions.proto.Function.SourceSpec; -import org.apache.pulsar.functions.proto.Function.SubscriptionType; -import org.apache.pulsar.functions.runtime.RuntimeFactory; -import org.apache.pulsar.functions.source.TopicSchema; -import org.apache.pulsar.functions.utils.FunctionCommon; +import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.utils.FunctionConfigUtils; -import org.apache.pulsar.functions.utils.functions.FunctionArchive; -import org.apache.pulsar.functions.utils.io.Connector; -import org.apache.pulsar.functions.worker.ConnectorsManager; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.FunctionsManager; -import org.apache.pulsar.functions.worker.LeaderService; -import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl; -import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; -import org.apache.pulsar.functions.worker.rest.api.v2.FunctionsApiV2Resource; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -/** - * Unit test of {@link FunctionsApiV2Resource}. - */ -public class FunctionApiV3ResourceTest { - - private static final class TestFunction implements Function { - - @Override - public String process(String input, Context context) { - return input; - } - } - - private static final class WrongFunction implements Consumer { - @Override - public void accept(String s) { - - } - } - - private static final String tenant = "test-tenant"; - private static final String namespace = "test-namespace"; - private static final String function = "test-function"; - private static final String outputTopic = "test-output-topic"; - private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE; - private static final String className = TestFunction.class.getName(); - private SubscriptionType subscriptionType = SubscriptionType.FAILOVER; - private static final Map topicsToSerDeClassName = new HashMap<>(); - static { - topicsToSerDeClassName.put("persistent://public/default/test_src", TopicSchema.DEFAULT_SERDE); - } - private static final int parallelism = 1; - - private PulsarWorkerService mockedWorkerService; - private PulsarAdmin mockedPulsarAdmin; - private Tenants mockedTenants; - private Namespaces mockedNamespaces; - private Functions mockedFunctions; - private TenantInfoImpl mockedTenantInfo; - private List namespaceList = new LinkedList<>(); - private FunctionMetaDataManager mockedManager; - private FunctionRuntimeManager mockedFunctionRunTimeManager; - private RuntimeFactory mockedRuntimeFactory; - private Namespace mockedNamespace; +public class FunctionApiV3ResourceTest extends AbstractFunctionApiResourceTest { private FunctionsImpl resource; - private InputStream mockedInputStream; - private FormDataContentDisposition mockedFormData; - private FunctionMetaData mockedFunctionMetadata; - private LeaderService mockedLeaderService; - private Packages mockedPackages; - private PulsarFunctionTestTemporaryDirectory tempDirectory; - private static Map mockStaticContexts = new HashMap<>(); - - private static final String SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH = - "pulsar-functions-api-examples.nar.path"; - - public static File getPulsarApiExamplesNar() { - return new File(Objects.requireNonNull( - System.getProperty(SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH) - , "pulsar-functions-api-examples.nar file location must be specified with " - + SYSTEM_PROPERTY_NAME_FUNCTIONS_API_EXAMPLES_NAR_FILE_PATH + " system property")); - } - - @BeforeMethod - public void setup() throws Exception { - this.mockedManager = mock(FunctionMetaDataManager.class); - this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); - this.mockedTenantInfo = mock(TenantInfoImpl.class); - this.mockedRuntimeFactory = mock(RuntimeFactory.class); - this.mockedInputStream = mock(InputStream.class); - this.mockedNamespace = mock(Namespace.class); - this.mockedFormData = mock(FormDataContentDisposition.class); - when(mockedFormData.getFileName()).thenReturn("test"); - this.mockedPulsarAdmin = mock(PulsarAdmin.class); - this.mockedTenants = mock(Tenants.class); - this.mockedNamespaces = mock(Namespaces.class); - this.mockedFunctions = mock(Functions.class); - this.mockedPackages = mock(Packages.class); - this.mockedLeaderService = mock(LeaderService.class); - this.mockedFunctionMetadata = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - namespaceList.add(tenant + "/" + namespace); - - this.mockedWorkerService = mock(PulsarWorkerService.class); - when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); - when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); - when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); - when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); - when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); - when(mockedWorkerService.isInitialized()).thenReturn(true); - when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); - when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); - when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); - when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages); - when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); - when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); - when(mockedLeaderService.isLeader()).thenReturn(true); - when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetadata); - doNothing().when(mockedPackages).download(anyString(), anyString()); - - // worker config - WorkerConfig workerConfig = new WorkerConfig() - .setWorkerId("test") - .setWorkerPort(8080) - .setFunctionMetadataTopicName("pulsar/functions") - .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); - tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); - tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); - when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); - + @Override + protected void doSetup() { + super.doSetup(); this.resource = spy(new FunctionsImpl(() -> mockedWorkerService)); } - @AfterMethod(alwaysRun = true) - public void cleanup() { - if (tempDirectory != null) { - tempDirectory.delete(); - } - mockStaticContexts.values().forEach(MockedStatic::close); - mockStaticContexts.clear(); - } - - private void mockStatic(Class classStatic, Consumer> consumer) { - final MockedStatic mockedStatic = mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); - consumer.accept(mockedStatic); - } - - private void mockWorkerUtils() { - mockWorkerUtils(null); - } - - private void mockWorkerUtils(Consumer> consumer) { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - if (consumer != null) { - consumer.accept(ctx); - } - }); - } - - private void mockInstanceUtils() { - mockStatic(InstanceUtils.class, ctx -> { - ctx.when(() -> InstanceUtils.calculateSubjectType(any())) - .thenReturn(FunctionDetails.ComponentType.FUNCTION); - }); - } - - // - // Register Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testRegisterFunctionMissingTenant() { - try { - testRegisterFunctionMissingArguments( - null, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testRegisterFunctionMissingNamespace() { - try { - testRegisterFunctionMissingArguments( - tenant, - null, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testRegisterFunctionMissingFunctionName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - null, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package is not provided") - public void testRegisterFunctionMissingPackage() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "No input topic\\(s\\) specified for the function") - public void testRegisterFunctionMissingInputTopics() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function Package is not provided") - public void testRegisterFunctionMissingPackageDetails() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - null, - outputTopic, - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function package does not have" - + " the correct format. Pulsar cannot determine if the package is a NAR package or JAR package. Function " - + "classname is not provided and attempts to load it as a NAR package produced the following error.*") - public void testRegisterFunctionMissingClassName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class UnknownClass must be in class path") - public void testRegisterFunctionWrongClassName() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - "UnknownClass", - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function parallelism must be a positive number") - public void testRegisterFunctionWrongParallelism() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - -2, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, - expectedExceptionsMessageRegExp = "Output topic persistent://public/default/test_src is also being used as an input topic \\(topics must be one or the other\\)") - public void testRegisterFunctionSameInputOutput() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - topicsToSerDeClassName.keySet().iterator().next(), - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Output topic " + function + "-output-topic/test:" + " is invalid") - public void testRegisterFunctionWrongOutputTopic() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - function + "-output-topic/test:", - outputSerdeClassName, - className, - parallelism, - null); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Encountered error .*. when getting Function package from .*") - public void testRegisterFunctionHttpUrl() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - null, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "http://localhost:1234/test"); - } catch (RestException re){ - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function class .*. does not implement the correct interface") - public void testRegisterFunctionImplementWrongInterface() { - try { - testRegisterFunctionMissingArguments( - tenant, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - WrongFunction.class.getName(), - parallelism, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testRegisterFunctionMissingArguments( - String tenant, - String namespace, - String function, - InputStream inputStream, - Map topicsToSerDeClassName, - FormDataContentDisposition details, - String outputTopic, - String outputSerdeClassName, - String className, - Integer parallelism, - String functionPkgUrl) { - FunctionConfig functionConfig = new FunctionConfig(); - if (tenant != null) { - functionConfig.setTenant(tenant); - } - if (namespace != null) { - functionConfig.setNamespace(namespace); - } - if (function != null) { - functionConfig.setName(function); - } - if (topicsToSerDeClassName != null) { - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - } - if (outputTopic != null) { - functionConfig.setOutput(outputTopic); - } - if (outputSerdeClassName != null) { - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - } - if (className != null) { - functionConfig.setClassName(className); - } - if (parallelism != null) { - functionConfig.setParallelism(parallelism); - } - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - - resource.registerFunction( - tenant, - namespace, - function, - inputStream, - details, - functionPkgUrl, - functionConfig, - null); - - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided") - public void testMissingFunctionConfig() { - resource.registerFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - null, - null, - null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided") - public void testUpdateMissingFunctionConfig() { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - resource.updateFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - null, - null, - null, null); - } - - @Test - public void testUpdateSourceWithNoChange() throws ClassNotFoundException { - mockWorkerUtils(); - - FunctionDetails functionDetails = createDefaultFunctionDetails(); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class}); - ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(),any(),any(),any())).thenCallRealMethod(); - ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - }); - - doReturn(Function.class).when(mockedClassLoader).loadClass(anyString()); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath()); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - // No change on config, - FunctionConfig funcConfig = createDefaultFunctionConfig(); - mockStatic(FunctionConfigUtils.class, ctx -> { - ctx.when(() -> FunctionConfigUtils.convertFromDetails(any())).thenReturn(funcConfig); - ctx.when(() -> FunctionConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionConfigUtils.convert(any(FunctionConfig.class), any(ClassLoader.class))).thenReturn(functionDetails); - ctx.when(() -> FunctionConfigUtils.convert(any(FunctionConfig.class), any(FunctionConfigUtils.ExtractedFunctionDetails.class))).thenReturn(functionDetails); - ctx.when(() -> FunctionConfigUtils.validateJavaFunction(any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionConfigUtils.doCommonChecks(any())).thenCallRealMethod(); - ctx.when(() -> FunctionConfigUtils.collectAllInputTopics(any())).thenCallRealMethod(); - ctx.when(() -> FunctionConfigUtils.doJavaChecks(any(), any())).thenCallRealMethod(); - }); - - // config has not changes and don't update auth, should fail - try { - resource.updateFunction( - funcConfig.getTenant(), - funcConfig.getNamespace(), - funcConfig.getName(), - null, - mockedFormData, - null, - funcConfig, - null, - null); - fail("Update without changes should fail"); - } catch (RestException e) { - assertTrue(e.getMessage().contains("Update contains no change")); - } - - try { - UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); - updateOptions.setUpdateAuthData(false); - resource.updateFunction( - funcConfig.getTenant(), - funcConfig.getNamespace(), - funcConfig.getName(), - null, - mockedFormData, - null, - funcConfig, - null, - updateOptions); - fail("Update without changes should fail"); - } catch (RestException e) { - assertTrue(e.getMessage().contains("Update contains no change")); - } - - // no changes but set the auth-update flag to true, should not fail - UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); - updateOptions.setUpdateAuthData(true); - resource.updateFunction( - funcConfig.getTenant(), - funcConfig.getNamespace(), - funcConfig.getName(), - null, - mockedFormData, - null, - funcConfig, - null, - updateOptions); - } - - - private void registerDefaultFunction() { - registerDefaultFunctionWithPackageUrl(null); - } - - private void registerDefaultFunctionWithPackageUrl(String packageUrl) { - FunctionConfig functionConfig = createDefaultFunctionConfig(); - resource.registerFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - packageUrl, - functionConfig, - null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function already exists") - public void testRegisterExistedFunction() { - try { - Configurator.setRootLevel(Level.DEBUG); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") - public void testRegisterFunctionUploadFailure() throws Exception { - try { - mockWorkerUtils(ctx -> { - ctx.when(() -> { - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - } - ).thenThrow(new IOException("upload failure")); - ; - }); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - @Test - public void testRegisterFunctionSuccess() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(timeOut = 20000) - public void testRegisterFunctionSuccessWithPackageName() { - registerDefaultFunctionWithPackageUrl("function://public/default/test@v1"); - } - - @Test(timeOut = 20000) - public void testRegisterFunctionFailedWithWrongPackageName() throws PulsarAdminException { - try { - doThrow(new PulsarAdminException("package name is invalid")) - .when(mockedPackages).download(anyString(), anyString()); - registerDefaultFunctionWithPackageUrl("function://"); - } catch (RestException e) { - // expected exception - assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace does not exist") - public void testRegisterFunctionNonExistingNamespace() { - try { - this.namespaceList.clear(); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant does not exist") - public void testRegisterFunctionNonexistantTenant() throws Exception { - try { - when(mockedTenants.getTenantInfo(any())).thenThrow(PulsarAdminException.NotFoundException.class); - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") - public void testRegisterFunctionFailure() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - doThrow(new IllegalArgumentException("function failed to register")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registration interrupted") - public void testRegisterFunctionInterrupted() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - doThrow(new IllegalStateException("Function registration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - registerDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - /* - Externally managed runtime, - uploadBuiltinSinksSources == false - Make sure uploadFileToBookkeeper is not called - */ - @Test - public void testRegisterFunctionSuccessK8sNoUpload() throws Exception { - mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(false); - - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenThrow(new RuntimeException("uploadFileToBookkeeper triggered")); - - }); - - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class}); - ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - - }); - - doReturn(Function.class).when(mockedClassLoader).loadClass(anyString()); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath()); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); - - when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - FunctionConfig functionConfig = createDefaultFunctionConfig(); - functionConfig.setJar("builtin://exclamation"); - - try (FileInputStream inputStream = new FileInputStream(getPulsarApiExamplesNar())) { - resource.registerFunction( - tenant, - namespace, - function, - inputStream, - mockedFormData, - null, - functionConfig, - null); - } - } - - /* - Externally managed runtime, - uploadBuiltinSinksSources == true - Make sure uploadFileToBookkeeper is called - */ - @Test - public void testRegisterFunctionSuccessK8sWithUpload() throws Exception { - final String injectedErrMsg = "uploadFileToBookkeeper triggered"; - mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(true); - - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class))) - .thenThrow(new RuntimeException(injectedErrMsg)); - - }); - - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getFunctionTypes(any(FunctionConfig.class), any(Class.class))).thenReturn(new Class[]{String.class, String.class}); - ctx.when(() -> FunctionCommon.convertRuntime(any(FunctionConfig.Runtime.class))).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - }); - - doReturn(Function.class).when(mockedClassLoader).loadClass(anyString()); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath()); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); - - when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - - FunctionConfig functionConfig = createDefaultFunctionConfig(); - functionConfig.setJar("builtin://exclamation"); - - try (FileInputStream inputStream = new FileInputStream(getPulsarApiExamplesNar())) { - try { - resource.registerFunction( - tenant, - namespace, - function, - inputStream, - mockedFormData, - null, - functionConfig, - null); - Assert.fail(); - } catch (RuntimeException e) { - Assert.assertEquals(e.getMessage(), injectedErrMsg); - } - } - } - - // - // Update Functions - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testUpdateFunctionMissingTenant() throws Exception { - try { - testUpdateFunctionMissingArguments( - null, - namespace, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Tenant is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testUpdateFunctionMissingNamespace() throws Exception { - try { - testUpdateFunctionMissingArguments( - tenant, - null, - function, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Namespace is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testUpdateFunctionMissingFunctionName() throws Exception { - try { - testUpdateFunctionMissingArguments( - tenant, - namespace, - null, - mockedInputStream, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Function name is not provided"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingPackage() throws Exception { - try { - mockWorkerUtils(); - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingInputTopic() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - className, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Update contains no change") - public void testUpdateFunctionMissingClassName() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - "Update contains no change"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test - public void testUpdateFunctionChangedParallelism() throws Exception { - try { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism + 1, - null); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test - public void testUpdateFunctionChangedInputs() throws Exception { - mockWorkerUtils(); - - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - topicsToSerDeClassName, - mockedFormData, - "DifferentOutput", - outputSerdeClassName, - null, - parallelism, - null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Input Topics cannot be altered") - public void testUpdateFunctionChangedOutput() throws Exception { - try { - mockWorkerUtils(); - - Map someOtherInput = new HashMap<>(); - someOtherInput.put("DifferentTopic", TopicSchema.DEFAULT_SERDE); - testUpdateFunctionMissingArguments( - tenant, - namespace, - function, - null, - someOtherInput, - mockedFormData, - outputTopic, - outputSerdeClassName, - null, - parallelism, - "Input Topics cannot be altered"); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testUpdateFunctionMissingArguments( - String tenant, - String namespace, - String function, - InputStream inputStream, - Map topicsToSerDeClassName, - FormDataContentDisposition details, - String outputTopic, - String outputSerdeClassName, - String className, - Integer parallelism, - String expectedError) throws Exception { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - FunctionConfig functionConfig = new FunctionConfig(); - if (tenant != null) { - functionConfig.setTenant(tenant); - } - if (namespace != null) { - functionConfig.setNamespace(namespace); - } - if (function != null) { - functionConfig.setName(function); - } - if (topicsToSerDeClassName != null) { - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - } - if (outputTopic != null) { - functionConfig.setOutput(outputTopic); - } - if (outputSerdeClassName != null) { - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - } - if (className != null) { - functionConfig.setClassName(className); - } - if (parallelism != null) { - functionConfig.setParallelism(parallelism); - } - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - - if (expectedError != null) { - doThrow(new IllegalArgumentException(expectedError)) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - } - - resource.updateFunction( - tenant, - namespace, - function, - inputStream, - details, - null, - functionConfig, - null, null); - - } - - private void updateDefaultFunction() { - updateDefaultFunctionWithPackageUrl(null); - } - - private void updateDefaultFunctionWithPackageUrl(String packageUrl) { - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - - resource.updateFunction( - tenant, - namespace, - function, - mockedInputStream, - mockedFormData, - packageUrl, - functionConfig, - null, null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testUpdateNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "upload failure") - public void testUpdateFunctionUploadFailure() throws Exception { - try { - mockWorkerUtils(ctx -> { - ctx.when(() -> { - WorkerUtils.uploadFileToBookkeeper( - anyString(), - any(File.class), - any(Namespace.class)); - - }).thenThrow(new IOException("upload failure")); - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - }); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - @Test - public void testUpdateFunctionSuccess() throws Exception { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - updateDefaultFunction(); - } - - @Test - public void testUpdateFunctionWithUrl() { - Configurator.setRootLevel(Level.DEBUG); - - String fileLocation = FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - String filePackageUrl = "file://" + fileLocation; - - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - resource.updateFunction( - tenant, - namespace, - function, - null, - null, - filePackageUrl, - functionConfig, - null, null); - - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to register") - public void testUpdateFunctionFailure() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalArgumentException("function failed to register")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function registeration interrupted") - public void testUpdateFunctionInterrupted() throws Exception { - try { - mockWorkerUtils(); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalStateException("Function registeration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - updateDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - - @Test(timeOut = 20000) - public void testUpdateFunctionSuccessWithPackageName() { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - updateDefaultFunctionWithPackageUrl("function://public/default/test@v1"); - } - - @Test(timeOut = 20000) - public void testUpdateFunctionFailedWithWrongPackageName() throws PulsarAdminException { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - try { - doThrow(new PulsarAdminException("package name is invalid")) - .when(mockedPackages).download(anyString(), anyString()); - registerDefaultFunctionWithPackageUrl("function://"); - } catch (RestException e) { - // expected exception - assertEquals(e.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - } - } - - // - // deregister function - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testDeregisterFunctionMissingTenant() { - try { - - testDeregisterFunctionMissingArguments( - null, - namespace, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testDeregisterFunctionMissingNamespace() { - try { - testDeregisterFunctionMissingArguments( - tenant, - null, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testDeregisterFunctionMissingFunctionName() { - try { - testDeregisterFunctionMissingArguments( - tenant, - namespace, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testDeregisterFunctionMissingArguments( - String tenant, - String namespace, - String function - ) { - resource.deregisterFunction( - tenant, - namespace, - function, - null); - } - - private void deregisterDefaultFunction() { - resource.deregisterFunction( - tenant, - namespace, - function, - null); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testDeregisterNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); - throw re; - } - } - - @Test - public void testDeregisterFunctionSuccess() { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - deregisterDefaultFunction(); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "function failed to deregister") - public void testDeregisterFunctionFailure() throws Exception { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalArgumentException("function failed to deregister")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function deregisteration interrupted") - public void testDeregisterFunctionInterrupted() throws Exception { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - doThrow(new IllegalStateException("Function deregisteration interrupted")) - .when(mockedManager).updateFunctionOnLeader(any(FunctionMetaData.class), Mockito.anyBoolean()); - - deregisterDefaultFunction(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); - throw re; - } - } - - // - // Get Function Info - // - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testGetFunctionMissingTenant() { - try { - testGetFunctionMissingArguments( - null, - namespace, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testGetFunctionMissingNamespace() { - try { - testGetFunctionMissingArguments( - tenant, - null, - function - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function name is not provided") - public void testGetFunctionMissingFunctionName() { - try { - testGetFunctionMissingArguments( - tenant, - namespace, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } - } - - private void testGetFunctionMissingArguments( - String tenant, - String namespace, - String function - ) { - resource.getFunctionInfo( - tenant, - namespace, - function,null - ); - - } - - private FunctionConfig getDefaultFunctionInfo() { - return resource.getFunctionInfo( - tenant, - namespace, - function, - null - ); - } - - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't exist") - public void testGetNotExistedFunction() { - try { - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - getDefaultFunctionInfo(); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); - throw re; - } - } - - @Test - public void testGetFunctionSuccess() { - mockInstanceUtils(); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - - SinkSpec sinkSpec = SinkSpec.newBuilder() - .setTopic(outputTopic) - .setSerDeClassName(outputSerdeClassName).build(); - FunctionDetails functionDetails = FunctionDetails.newBuilder() - .setClassName(className) - .setSink(sinkSpec) - .setAutoAck(true) - .setName(function) - .setNamespace(namespace) - .setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE) - .setTenant(tenant) - .setParallelism(parallelism) - .setSource(SourceSpec.newBuilder().setSubscriptionType(subscriptionType) - .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build(); - FunctionMetaData metaData = FunctionMetaData.newBuilder() - .setCreateTime(System.currentTimeMillis()) - .setFunctionDetails(functionDetails) - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) - .setVersion(1234) - .build(); - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - - FunctionConfig functionConfig = getDefaultFunctionInfo(); - assertEquals( - FunctionConfigUtils.convertFromDetails(functionDetails), - functionConfig); - } - - // - // List Functions - // + protected void registerFunction(String tenant, String namespace, String function, InputStream inputStream, + FormDataContentDisposition details, String functionPkgUrl, FunctionConfig functionConfig) { + resource.registerFunction( + tenant, + namespace, + function, + inputStream, + details, + functionPkgUrl, + functionConfig, + null); + } + protected void updateFunction(String tenant, + String namespace, + String functionName, + InputStream uploadedInputStream, + FormDataContentDisposition fileDetail, + String functionPkgUrl, + FunctionConfig functionConfig, + AuthenticationParameters authParams, + UpdateOptionsImpl updateOptions) { + resource.updateFunction(tenant, namespace, functionName, uploadedInputStream, fileDetail, functionPkgUrl, + functionConfig, authParams, updateOptions); + } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") - public void testListFunctionsMissingTenant() { - try { - testListFunctionsMissingArguments( - null, - namespace - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; - } + protected StreamingOutput downloadFunction(String tenant, String namespace, String componentName, + AuthenticationParameters authParams, boolean transformFunction) { + return resource.downloadFunction(tenant, namespace, componentName, authParams, transformFunction); } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Namespace is not provided") - public void testListFunctionsMissingNamespace() { - try { - testListFunctionsMissingArguments( - tenant, - null - ); - } catch (RestException re) { - assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); - throw re; + protected File downloadFunction(final String path, final AuthenticationParameters authParams) throws IOException { + StreamingOutput streamingOutput = resource.downloadFunction(path, authParams); + File pkgFile = File.createTempFile("testpkg", "nar"); + try(OutputStream output = new FileOutputStream(pkgFile)) { + streamingOutput.write(output); } + return pkgFile; } - private void testListFunctionsMissingArguments( + protected void testDeregisterFunctionMissingArguments( String tenant, - String namespace + String namespace, + String function ) { - resource.listFunctions( - tenant, - namespace,null - ); - - } - - private List listDefaultFunctions() { - return resource.listFunctions( - tenant, - namespace,null - ); - } - - @Test - public void testListFunctionsSuccess() { - mockInstanceUtils(); - final List functions = Lists.newArrayList("test-1", "test-2"); - final List metaDataList = new LinkedList<>(); - FunctionMetaData functionMetaData1 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-1").build() - ).build(); - FunctionMetaData functionMetaData2 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder().setName("test-2").build() - ).build(); - metaDataList.add(functionMetaData1); - metaDataList.add(functionMetaData2); - when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(metaDataList); - - List functionList = listDefaultFunctions(); - assertEquals(functions, functionList); + resource.deregisterFunction( + tenant, + namespace, + function, + null); } - @Test - public void testOnlyGetSources() { - List functions = Lists.newArrayList("test-2"); - List functionMetaDataList = new LinkedList<>(); - FunctionMetaData f1 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder() - .setName("test-1") - .setComponentType(FunctionDetails.ComponentType.SOURCE) - .build()).build(); - functionMetaDataList.add(f1); - FunctionMetaData f2 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder() - .setName("test-2") - .setComponentType(FunctionDetails.ComponentType.FUNCTION) - .build()).build(); - functionMetaDataList.add(f2); - FunctionMetaData f3 = FunctionMetaData.newBuilder().setFunctionDetails( - FunctionDetails.newBuilder() - .setName("test-3") - .setComponentType(FunctionDetails.ComponentType.SINK) - .build()).build(); - functionMetaDataList.add(f3); - when(mockedManager.listFunctions(eq(tenant), eq(namespace))).thenReturn(functionMetaDataList); - - List functionList = listDefaultFunctions(); - assertEquals(functions, functionList); + protected void deregisterDefaultFunction() { + resource.deregisterFunction( + tenant, + namespace, + function, + null); } - @Test - public void testDownloadFunctionHttpUrl() throws Exception { - String jarHttpUrl = - "https://repo1.maven.org/maven2/org/apache/pulsar/pulsar-common/2.4.2/pulsar-common-2.4.2.jar"; - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + protected void testGetFunctionMissingArguments( + String tenant, + String namespace, + String function + ) { + resource.getFunctionInfo( + tenant, + namespace, + function, null + ); - StreamingOutput streamOutput = resource.downloadFunction(jarHttpUrl, null); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - Assert.assertTrue(pkgFile.exists()); - pkgFile.delete(); } - @Test - public void testDownloadFunctionFile() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - - StreamingOutput streamOutput = resource.downloadFunction("file:///" + fileLocation, null); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - Assert.assertTrue(pkgFile.exists()); - Assert.assertEquals(file.length(), pkgFile.length()); - pkgFile.delete(); + protected FunctionConfig getDefaultFunctionInfo() { + return resource.getFunctionInfo( + tenant, + namespace, + function, + null + ); } - @Test - public void testDownloadFunctionBuiltinConnector() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - - WorkerConfig config = new WorkerConfig() - .setUploadBuiltinSinksSources(false); - when(mockedWorkerService.getWorkerConfig()).thenReturn(config); - - Connector connector = Connector.builder().archivePath(file.toPath()).build(); - ConnectorsManager connectorsManager = mock(ConnectorsManager.class); - when(connectorsManager.getConnector("cassandra")).thenReturn(connector); - when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); - - StreamingOutput streamOutput = resource.downloadFunction("builtin://cassandra", null); + protected void testListFunctionsMissingArguments( + String tenant, + String namespace + ) { + resource.listFunctions( + tenant, + namespace, null + ); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - output.flush(); - output.close(); - Assert.assertTrue(pkgFile.exists()); - Assert.assertTrue(pkgFile.exists()); - Assert.assertEquals(file.length(), pkgFile.length()); - pkgFile.delete(); } - @Test - public void testDownloadFunctionBuiltinFunction() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); - - WorkerConfig config = new WorkerConfig() - .setUploadBuiltinSinksSources(false); - when(mockedWorkerService.getWorkerConfig()).thenReturn(config); - - FunctionsManager functionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder().archivePath(file.toPath()).build(); - when(functionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); - when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); - - StreamingOutput streamOutput = resource.downloadFunction("builtin://exclamation", null); - - File pkgFile = new File(testDir, UUID.randomUUID().toString()); - OutputStream output = new FileOutputStream(pkgFile); - streamOutput.write(output); - output.flush(); - output.close(); - Assert.assertTrue(pkgFile.exists()); - Assert.assertEquals(file.length(), pkgFile.length()); - pkgFile.delete(); + protected List listDefaultFunctions() { + return resource.listFunctions( + tenant, + namespace, null + ); } @Test public void testDownloadFunctionBuiltinConnectorByName() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + File file = getPulsarApiExamplesNar(); WorkerConfig config = new WorkerConfig() - .setUploadBuiltinSinksSources(false); + .setUploadBuiltinSinksSources(false); when(mockedWorkerService.getWorkerConfig()).thenReturn(config); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - FunctionMetaData metaData = FunctionMetaData.newBuilder() - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("builtin://cassandra")) - .setTransformFunctionPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) - .setFunctionDetails(FunctionDetails.newBuilder().setComponentType(FunctionDetails.ComponentType.SINK)) + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("builtin://cassandra")) + .setTransformFunctionPackageLocation( + Function.PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .setFunctionDetails(Function.FunctionDetails.newBuilder().setComponentType(Function.FunctionDetails.ComponentType.SINK)) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - Connector connector = Connector.builder().archivePath(file.toPath()).build(); - ConnectorsManager connectorsManager = mock(ConnectorsManager.class); - when(connectorsManager.getConnector("cassandra")).thenReturn(connector); - when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); + registerBuiltinConnector("cassandra", file); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + StreamingOutput streamOutput = downloadFunction(tenant, namespace, function, AuthenticationParameters.builder().build(), false); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); + File pkgFile = File.createTempFile("testpkg", "nar"); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); Assert.assertTrue(pkgFile.exists()); @@ -1812,31 +189,27 @@ public void testDownloadFunctionBuiltinConnectorByName() throws Exception { @Test public void testDownloadFunctionBuiltinFunctionByName() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + File file = getPulsarApiExamplesNar(); WorkerConfig config = new WorkerConfig() - .setUploadBuiltinSinksSources(false); + .setUploadBuiltinSinksSources(false); when(mockedWorkerService.getWorkerConfig()).thenReturn(config); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - FunctionMetaData metaData = FunctionMetaData.newBuilder() - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("builtin://exclamation")) - .setTransformFunctionPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) - .setFunctionDetails(FunctionDetails.newBuilder().setComponentType(FunctionDetails.ComponentType.FUNCTION)) - .build(); + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("builtin://exclamation")) + .setTransformFunctionPackageLocation( + Function.PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .setFunctionDetails( + Function.FunctionDetails.newBuilder().setComponentType(Function.FunctionDetails.ComponentType.FUNCTION)) + .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - FunctionsManager functionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder().archivePath(file.toPath()).build(); - when(functionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); - when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); + registerBuiltinFunction("exclamation", file); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + StreamingOutput streamOutput = downloadFunction(tenant, namespace, function, AuthenticationParameters.builder().build(), false); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); + File pkgFile = File.createTempFile("testpkg", "nar"); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); Assert.assertTrue(pkgFile.exists()); @@ -1846,32 +219,26 @@ public void testDownloadFunctionBuiltinFunctionByName() throws Exception { @Test public void testDownloadTransformFunctionByName() throws Exception { - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String testDir = FunctionApiV3ResourceTest.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + File file = getPulsarApiExamplesNar(); WorkerConfig workerConfig = new WorkerConfig() - .setUploadBuiltinSinksSources(false); + .setUploadBuiltinSinksSources(false); when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - FunctionMetaData metaData = FunctionMetaData.newBuilder() - .setPackageLocation(PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) - .setTransformFunctionPackageLocation(PackageLocationMetaData.newBuilder() + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("http://invalid")) + .setTransformFunctionPackageLocation(Function.PackageLocationMetaData.newBuilder() .setPackagePath("builtin://exclamation")) .build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); - FunctionsManager functionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder().archivePath(file.toPath()).build(); - when(functionsManager.getFunction("exclamation")).thenReturn(functionArchive); - when(mockedWorkerService.getConnectorsManager()).thenReturn(mock(ConnectorsManager.class)); - when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); + registerBuiltinFunction("exclamation", file); - StreamingOutput streamOutput = resource.downloadFunction(tenant, namespace, function, + StreamingOutput streamOutput = downloadFunction(tenant, namespace, function, AuthenticationParameters.builder().build(), true); - File pkgFile = new File(testDir, UUID.randomUUID().toString()); + File pkgFile = File.createTempFile("testpkg", "nar"); OutputStream output = new FileOutputStream(pkgFile); streamOutput.write(output); Assert.assertTrue(pkgFile.exists()); @@ -1879,15 +246,58 @@ public void testDownloadTransformFunctionByName() throws Exception { pkgFile.delete(); } + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function test-function doesn't" + + " exist") + public void testGetNotExistedFunction() throws IOException { + try { + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); + getDefaultFunctionInfo(); + } catch (RestException re) { + assertEquals(re.getResponse().getStatusInfo(), Response.Status.NOT_FOUND); + throw re; + } + } @Test - public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { + public void testGetFunctionSuccess() throws IOException { + mockInstanceUtils(); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); + + Function.SinkSpec sinkSpec = Function.SinkSpec.newBuilder() + .setTopic(outputTopic) + .setSerDeClassName(outputSerdeClassName).build(); + Function.FunctionDetails functionDetails = Function.FunctionDetails.newBuilder() + .setClassName(className) + .setSink(sinkSpec) + .setAutoAck(true) + .setName(function) + .setNamespace(namespace) + .setProcessingGuarantees(Function.ProcessingGuarantees.ATMOST_ONCE) + .setTenant(tenant) + .setParallelism(parallelism) + .setSource(Function.SourceSpec.newBuilder().setSubscriptionType(subscriptionType) + .putAllTopicsToSerDeClassName(topicsToSerDeClassName)).build(); + Function.FunctionMetaData metaData = Function.FunctionMetaData.newBuilder() + .setCreateTime(System.currentTimeMillis()) + .setFunctionDetails(functionDetails) + .setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/path/to/package")) + .setVersion(1234) + .build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(function))).thenReturn(metaData); + + FunctionConfig functionConfig = getDefaultFunctionInfo(); + assertEquals( + FunctionConfigUtils.convertFromDetails(functionDetails), + functionConfig); + } + + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function language runtime is " + + "either not set or cannot be determined") + public void testCreateFunctionWithoutSettingRuntime() throws Exception { Configurator.setRootLevel(Level.DEBUG); - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; + File file = getPulsarApiExamplesNar(); + String filePackageUrl = file.toURI().toString(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); FunctionConfig functionConfig = new FunctionConfig(); @@ -1896,82 +306,135 @@ public void testRegisterFunctionFileUrlWithValidSinkClass() throws Exception { functionConfig.setName(function); functionConfig.setClassName(className); functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); functionConfig.setOutput(outputTopic); functionConfig.setOutputSerdeClassName(outputSerdeClassName); - resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null); + registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig); } @Test - public void testRegisterFunctionWithConflictingFields() throws Exception { - Configurator.setRootLevel(Level.DEBUG); - String actualTenant = "DIFFERENT_TENANT"; - String actualNamespace = "DIFFERENT_NAMESPACE"; - String actualName = "DIFFERENT_NAME"; - this.namespaceList.add(actualTenant + "/" + actualNamespace); + public void testUpdateSourceWithNoChange() throws IOException { + mockWorkerUtils(); - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); - when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); + FunctionConfig funcConfig = createDefaultFunctionConfig(); - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - resource.registerFunction(actualTenant, actualNamespace, actualName, null, null, filePackageUrl, functionConfig, - null); + // config has not changes and don't update auth, should fail + try { + updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + null); + fail("Update without changes should fail"); + } catch (RestException e) { + assertThat(e.getMessage()).contains("Update contains no change"); + } + + try { + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(false); + updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + updateOptions); + fail("Update without changes should fail"); + } catch (RestException e) { + assertTrue(e.getMessage().contains("Update contains no change")); + } + + // no changes but set the auth-update flag to true, should not fail + UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); + updateOptions.setUpdateAuthData(true); + updateFunction( + funcConfig.getTenant(), + funcConfig.getNamespace(), + funcConfig.getName(), + null, + mockedFormData, + null, + funcConfig, + null, + updateOptions); } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function language runtime is either not set or cannot be determined") - public void testCreateFunctionWithoutSettingRuntime() throws Exception { - Configurator.setRootLevel(Level.DEBUG); + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Function config is not provided") + public void testMissingFunctionConfig() throws IOException { + registerFunction(tenant, namespace, function, mockedInputStream, mockedFormData, null, null); + } + + /* + Externally managed runtime, + uploadBuiltinSinksSources == false + Make sure uploadFileToBookkeeper is not called + */ + @Test + public void testRegisterFunctionSuccessK8sNoUpload() throws Exception { + mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(false); + + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new RuntimeException("uploadFileToBookkeeper triggered")); + + }); - URL fileUrl = getClass().getClassLoader().getResource("test_worker_config.yml"); - File file = Paths.get(fileUrl.toURI()).toFile(); - String fileLocation = file.getAbsolutePath().replace('\\', '/'); - String filePackageUrl = "file:///" + fileLocation; + registerBuiltinFunction("exclamation", getPulsarApiExamplesNar()); + when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - resource.registerFunction(tenant, namespace, function, null, null, filePackageUrl, functionConfig, null); + FunctionConfig functionConfig = createDefaultFunctionConfig(); + functionConfig.setJar("builtin://exclamation"); + registerFunction(tenant, namespace, function, null, mockedFormData, null, functionConfig); } - public static FunctionConfig createDefaultFunctionConfig() { - FunctionConfig functionConfig = new FunctionConfig(); - functionConfig.setTenant(tenant); - functionConfig.setNamespace(namespace); - functionConfig.setName(function); - functionConfig.setClassName(className); - functionConfig.setParallelism(parallelism); - functionConfig.setCustomSerdeInputs(topicsToSerDeClassName); - functionConfig.setOutput(outputTopic); - functionConfig.setOutputSerdeClassName(outputSerdeClassName); - functionConfig.setRuntime(FunctionConfig.Runtime.JAVA); - return functionConfig; - } + /* + Externally managed runtime, + uploadBuiltinSinksSources == true + Make sure uploadFileToBookkeeper is called + */ + @Test + public void testRegisterFunctionSuccessK8sWithUpload() throws Exception { + final String injectedErrMsg = "uploadFileToBookkeeper triggered"; + mockedWorkerService.getWorkerConfig().setUploadBuiltinSinksSources(true); + + mockStatic(WorkerUtils.class, ctx -> { + ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( + anyString(), + any(File.class), + any(Namespace.class))) + .thenThrow(new RuntimeException(injectedErrMsg)); + + }); + + registerBuiltinFunction("exclamation", getPulsarApiExamplesNar()); + when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); - public static FunctionDetails createDefaultFunctionDetails() { FunctionConfig functionConfig = createDefaultFunctionConfig(); - return FunctionConfigUtils.convert(functionConfig, (ClassLoader) null); + functionConfig.setJar("builtin://exclamation"); + + try { + registerFunction(tenant, namespace, function, null, mockedFormData, null, functionConfig); + Assert.fail(); + } catch (RuntimeException e) { + Assert.assertEquals(e.getMessage(), injectedErrMsg); + } } + } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 5dcc795304ef5..b9833380d7087 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -18,246 +18,78 @@ */ package org.apache.pulsar.functions.worker.rest.api.v3; +import static org.apache.pulsar.functions.proto.Function.ProcessingGuarantees.ATLEAST_ONCE; +import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Lists; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.function.Consumer; import javax.ws.rs.core.Response; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.pulsar.broker.authentication.AuthenticationParameters; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.Packages; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Tenants; -import org.apache.pulsar.common.functions.FunctionConfig; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.io.SinkConfig; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; -import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; -import org.apache.pulsar.functions.api.examples.ExclamationFunction; import org.apache.pulsar.functions.api.examples.RecordFunction; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.instance.InstanceUtils; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.Function.FunctionMetaData; -import org.apache.pulsar.functions.runtime.RuntimeFactory; -import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SinkConfigUtils; -import org.apache.pulsar.functions.utils.functions.FunctionArchive; -import org.apache.pulsar.functions.utils.functions.FunctionUtils; -import org.apache.pulsar.functions.utils.io.Connector; -import org.apache.pulsar.functions.utils.io.ConnectorUtils; -import org.apache.pulsar.functions.worker.ConnectorsManager; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.FunctionsManager; -import org.apache.pulsar.functions.worker.LeaderService; -import org.apache.pulsar.functions.worker.PulsarWorkerService; -import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; -import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; import org.apache.pulsar.functions.worker.rest.api.SinksImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static org.apache.pulsar.functions.proto.Function.ProcessingGuarantees.ATLEAST_ONCE; -import static org.apache.pulsar.functions.source.TopicSchema.DEFAULT_SERDE; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; /** * Unit test of {@link SinksApiV3Resource}. */ -public class SinkApiV3ResourceTest { +public class SinkApiV3ResourceTest extends AbstractFunctionsResourceTest { - private static final String tenant = "test-tenant"; - private static final String namespace = "test-namespace"; private static final String sink = "test-sink"; - private static final Map topicsToSerDeClassName = new HashMap<>(); - - static { - topicsToSerDeClassName.put("test_src", DEFAULT_SERDE); - } - - private static final String subscriptionName = "test-subscription"; - private static final String CASSANDRA_STRING_SINK = "org.apache.pulsar.io.cassandra.CassandraStringSink"; - private static final int parallelism = 1; - - private PulsarWorkerService mockedWorkerService; - private PulsarAdmin mockedPulsarAdmin; - private Tenants mockedTenants; - private Namespaces mockedNamespaces; - private Functions mockedFunctions; - private TenantInfoImpl mockedTenantInfo; - private List namespaceList = new LinkedList<>(); - private FunctionMetaDataManager mockedManager; - private FunctionRuntimeManager mockedFunctionRunTimeManager; - private RuntimeFactory mockedRuntimeFactory; - private Namespace mockedNamespace; + private SinksImpl resource; - private InputStream mockedInputStream; - private FormDataContentDisposition mockedFormData; - private FunctionMetaData mockedFunctionMetaData; - private LeaderService mockedLeaderService; - private Packages mockedPackages; - private PulsarFunctionTestTemporaryDirectory tempDirectory; - private static Map mockStaticContexts = new HashMap<>(); - - private static final String SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH = "pulsar-io-cassandra.nar.path"; - - public static File getPulsarIOCassandraNar() { - return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH) - , "pulsar-io-cassandra.nar file location must be specified with " - + SYSTEM_PROPERTY_NAME_CASSANDRA_NAR_FILE_PATH + " system property")); - } - - private static final String SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH = "pulsar-io-twitter.nar.path"; - - public static File getPulsarIOTwitterNar() { - return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH) - , "pulsar-io-twitter.nar file location must be specified with " - + SYSTEM_PROPERTY_NAME_TWITTER_NAR_FILE_PATH + " system property")); - } - - private static final String SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH = "pulsar-io-invalid.nar.path"; - - public static File getPulsarIOInvalidNar() { - return new File(Objects.requireNonNull(System.getProperty(SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH) - , "invalid nar file location must be specified with " - + SYSTEM_PROPERTY_NAME_INVALID_NAR_FILE_PATH + " system property")); - } - - @BeforeMethod - public void setup() throws Exception { - this.mockedManager = mock(FunctionMetaDataManager.class); - this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); - this.mockedRuntimeFactory = mock(RuntimeFactory.class); - this.mockedInputStream = mock(InputStream.class); - this.mockedNamespace = mock(Namespace.class); - this.mockedFormData = mock(FormDataContentDisposition.class); - when(mockedFormData.getFileName()).thenReturn("test"); - this.mockedTenantInfo = mock(TenantInfoImpl.class); - this.mockedPulsarAdmin = mock(PulsarAdmin.class); - this.mockedTenants = mock(Tenants.class); - this.mockedNamespaces = mock(Namespaces.class); - this.mockedFunctions = mock(Functions.class); - this.mockedLeaderService = mock(LeaderService.class); - this.mockedPackages = mock(Packages.class); - namespaceList.add(tenant + "/" + namespace); - - this.mockedWorkerService = mock(PulsarWorkerService.class); - when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); - when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); - when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); - when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); - when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); - when(mockedWorkerService.isInitialized()).thenReturn(true); - when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); - when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); - when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); - when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages); - when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); - when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); - when(mockedLeaderService.isLeader()).thenReturn(true); - doAnswer(invocationOnMock -> { - Files.copy(getPulsarIOCassandraNar().toPath(), Paths.get(invocationOnMock.getArgument(1, String.class)), - StandardCopyOption.REPLACE_EXISTING); - return null; - }).when(mockedPackages).download(any(), any()); - - // worker config - WorkerConfig workerConfig = new WorkerConfig() - .setWorkerId("test") - .setWorkerPort(8080) - .setFunctionMetadataTopicName("pulsar/functions") - .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); - tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); - tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); - when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); + @Override + protected void doSetup() { this.resource = spy(new SinksImpl(() -> mockedWorkerService)); - } - @AfterMethod(alwaysRun = true) - public void cleanup() { - if (tempDirectory != null) { - tempDirectory.delete(); - } - mockStaticContexts.values().forEach(MockedStatic::close); - mockStaticContexts.clear(); + @Override + protected Function.FunctionDetails.ComponentType getComponentType() { + return Function.FunctionDetails.ComponentType.SINK; } - private void mockStatic(Class classStatic, Consumer> consumer) { - final MockedStatic mockedStatic = - mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); - consumer.accept(mockedStatic); - } - - private void mockWorkerUtils() { - mockWorkerUtils(null); - } - - private void mockWorkerUtils(Consumer> consumer) { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - if (consumer != null) { - consumer.accept(ctx); - } - }); + @Override + protected File getDefaultNarFile() { + return getPulsarIOCassandraNar(); } - private void mockInstanceUtils() { - mockStatic(InstanceUtils.class, ctx -> { - ctx.when(() -> InstanceUtils.calculateSubjectType(any())) - .thenReturn(FunctionDetails.ComponentType.SINK); - }); - } - - - // - // Register Functions - // - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Tenant is not provided") public void testRegisterSinkMissingTenant() { try { @@ -337,8 +169,8 @@ public void testRegisterSinkMissingPackage() { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink class UnknownClass must " - + "be in class path") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink class UnknownClass not " + + "found") public void testRegisterSinkWrongClassName() { mockInstanceUtils(); try { @@ -359,10 +191,8 @@ public void testRegisterSinkWrongClassName() { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink package does not have the" - + " correct format. Pulsar cannot determine if the package is a NAR package" - + " or JAR package. Sink classname is not provided and attempts to load it as a NAR package produced the " - + "following error.") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Sink package doesn't contain " + + "the META-INF/services/pulsar-io.yaml file.") public void testRegisterSinkMissingPackageDetails() { mockInstanceUtils(); try { @@ -722,30 +552,11 @@ public void testRegisterSinkSuccessWithTransformFunction() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - doReturn(RecordFunction.class).when(mockedClassLoader).loadClass("RecordFunction"); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); - - mockStatic(FunctionUtils.class, ctx -> { - ctx.when(() -> FunctionUtils.getFunctionClass(any())).thenReturn("RecordFunction"); - }); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); + registerBuiltinConnector("recordfunction", RecordFunction.class.getName()); + registerBuiltinFunction("transform", RecordFunction.class.getName()); SinkConfig sinkConfig = createDefaultSinkConfig(); + sinkConfig.setSinkType("builtin://recordfunction"); sinkConfig.setTransformFunction("builtin://transform"); sinkConfig.setTransformFunctionConfig("{\"dummy\": \"dummy\"}"); @@ -770,28 +581,7 @@ public void testRegisterSinkFailureWithInvalidTransformFunction() throws Excepti when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - doReturn(ExclamationFunction.class).when(mockedClassLoader).loadClass("ExclamationFunction"); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); - - mockStatic(FunctionUtils.class, ctx -> { - ctx.when(() -> FunctionUtils.getFunctionClass(any())).thenReturn("ExclamationFunction"); - }); - - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); + registerBuiltinFunction("transform", getPulsarApiExamplesNar()); SinkConfig sinkConfig = createDefaultSinkConfig(); sinkConfig.setTransformFunction("builtin://transform"); @@ -946,16 +736,18 @@ public void testUpdateSinkDifferentInputs() throws Exception { public void testUpdateSinkDifferentParallelism() throws Exception { mockWorkerUtils(); - testUpdateSinkMissingArguments( - tenant, - namespace, - sink, - null, - mockedFormData, - topicsToSerDeClassName, - CASSANDRA_STRING_SINK, - parallelism + 1, - null); + try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { + testUpdateSinkMissingArguments( + tenant, + namespace, + sink, + inputStream, + mockedFormData, + topicsToSerDeClassName, + CASSANDRA_STRING_SINK, + parallelism + 1, + null); + } } private void testUpdateSinkMissingArguments( @@ -1007,32 +799,6 @@ private void testUpdateSinkMissingArguments( } - private void mockFunctionCommon(String tenant, String namespace, String sink) throws IOException { - mockStatic(ConnectorUtils.class, ctx -> { - ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) - .thenReturn(CASSANDRA_STRING_SINK); - }); - - mockStatic(ClassLoaderUtils.class, ctx -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); - ctx.when(() -> FunctionCommon - .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) - .thenReturn(ATLEAST_ONCE); - }); - - this.mockedFunctionMetaData = - FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); - when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); - - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - } - private void updateDefaultSink() throws Exception { updateDefaultSinkWithPackageUrl(null); } @@ -1040,25 +806,6 @@ private void updateDefaultSink() throws Exception { private void updateDefaultSinkWithPackageUrl(String packageUrl) throws Exception { SinkConfig sinkConfig = createDefaultSinkConfig(); - mockStatic(ConnectorUtils.class, ctx -> { - ctx.when(() -> ConnectorUtils.getIOSinkClass(any(NarClassLoader.class))) - .thenReturn(CASSANDRA_STRING_SINK); - }); - - mockStatic(ClassLoaderUtils.class, ctx -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); - ctx.when(() -> FunctionCommon - .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) - .thenReturn(ATLEAST_ONCE); - }); - - this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); @@ -1091,7 +838,6 @@ public void testUpdateNotExistedSink() throws Exception { public void testUpdateSinkUploadFailure() throws Exception { try { mockWorkerUtils(ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); ctx.when(() -> WorkerUtils.uploadFileToBookkeeper( anyString(), any(File.class), @@ -1127,24 +873,6 @@ public void testUpdateSinkWithUrl() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - mockStatic(ConnectorUtils.class, ctx -> { - ctx.when(() -> ConnectorUtils.getIOSinkClass(any())) - .thenReturn(CASSANDRA_STRING_SINK); - }); - - mockStatic(ClassLoaderUtils.class, ctx -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.extractFileFromPkgURL(any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mock(NarClassLoader.class)); - ctx.when(() -> FunctionCommon - .convertProcessingGuarantee(eq(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE))) - .thenReturn(ATLEAST_ONCE); - }); - this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); @@ -1219,35 +947,17 @@ public void testUpdateSinkDifferentTransformFunction() throws Exception { SinkConfig sinkConfig = createDefaultSinkConfig(); sinkConfig.setTransformFunction("builtin://transform"); - sinkConfig.setTransformFunctionClassName("DummyFunction"); sinkConfig.setTransformFunctionConfig("{\"dummy\": \"dummy\"}"); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - doReturn(RecordFunction.class).when(mockedClassLoader).loadClass("DummyFunction"); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(FunctionCommon::createPkgTempFile).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getRawFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionTypes(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getFunctionClassParent(any(), anyBoolean())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); + registerBuiltinFunction("transform", RecordFunction.class.getName()); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); - when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); - FunctionsManager mockedFunctionsManager = mock(FunctionsManager.class); - FunctionArchive functionArchive = FunctionArchive.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedFunctionsManager.getFunction("transform")).thenReturn(functionArchive); - - when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); - - try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { resource.updateSink( tenant, @@ -1737,6 +1447,14 @@ private SinkConfig createDefaultSinkConfig() { return sinkConfig; } + private void mockFunctionCommon(String tenant, String namespace, String sink) throws IOException { + this.mockedFunctionMetaData = + Function.FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(eq(tenant), eq(namespace), eq(sink))).thenReturn(mockedFunctionMetaData); + + when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(true); + } + private FunctionDetails createDefaultFunctionDetails() throws IOException { return SinkConfigUtils.convert(createDefaultSinkConfig(), new SinkConfigUtils.ExtractedSinkDetails(null, null, null)); @@ -1760,21 +1478,7 @@ public void testRegisterSinkSuccessK8sNoUpload() throws Exception { }); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); - - ConnectorsManager mockedConnManager = mock(ConnectorsManager.class); - Connector connector = Connector.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedConnManager.getConnector("cassandra")).thenReturn(connector); - when(mockedConnManager.getSinkArchive(any())).thenReturn(getPulsarIOCassandraNar().toPath()); - when(mockedWorkerService.getConnectorsManager()).thenReturn(mockedConnManager); + registerBuiltinConnector("cassandra", getPulsarIOCassandraNar()); when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); @@ -1814,23 +1518,7 @@ public void testRegisterSinkSuccessK8sWithUpload() throws Exception { }); - NarClassLoader mockedClassLoader = mock(NarClassLoader.class); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.getSinkType(any())).thenReturn(String.class); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); - - ConnectorsManager mockedConnManager = mock(ConnectorsManager.class); - Connector connector = Connector.builder() - .classLoader(mockedClassLoader) - .build(); - when(mockedConnManager.getConnector("cassandra")).thenReturn(connector); - when(mockedConnManager.getSinkArchive(any())).thenReturn(getPulsarIOCassandraNar().toPath()); - - when(mockedWorkerService.getConnectorsManager()).thenReturn(mockedConnManager); - + registerBuiltinConnector("cassandra", getPulsarIOCassandraNar()); when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(sink))).thenReturn(false); @@ -1865,11 +1553,6 @@ public void testUpdateSinkWithNoChange() throws IOException { mockStatic(SinkConfigUtils.class, ctx -> { ctx.when(() -> SinkConfigUtils.convertFromDetails(any())).thenReturn(sinkConfig); - ctx.when(() -> SinkConfigUtils.convert(any(), any())).thenCallRealMethod(); - ctx.when(() -> SinkConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); - ctx.when(() -> SinkConfigUtils.clone(any())).thenCallRealMethod(); - ctx.when(() -> SinkConfigUtils.collectAllInputTopics(any())).thenCallRealMethod(); - ctx.when(() -> SinkConfigUtils.validateAndExtractDetails(any(),any(),any(),anyBoolean())).thenCallRealMethod(); }); mockFunctionCommon(sinkConfig.getTenant(), sinkConfig.getNamespace(), sinkConfig.getName()); @@ -1912,15 +1595,17 @@ public void testUpdateSinkWithNoChange() throws IOException { // no changes but set the auth-update flag to true, should not fail UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); updateOptions.setUpdateAuthData(true); - resource.updateSink( - sinkConfig.getTenant(), - sinkConfig.getNamespace(), - sinkConfig.getName(), - null, - mockedFormData, - null, - sinkConfig, - null, - updateOptions); + try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { + resource.updateSink( + sinkConfig.getTenant(), + sinkConfig.getNamespace(), + sinkConfig.getName(), + inputStream, + mockedFormData, + null, + sinkConfig, + null, + updateOptions); + } } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java index eabf954cc77b1..c7e69484d3019 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java @@ -18,17 +18,10 @@ */ package org.apache.pulsar.functions.worker.rest.api.v3; -import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOCassandraNar; -import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOInvalidNar; -import static org.apache.pulsar.functions.worker.rest.api.v3.SinkApiV3ResourceTest.getPulsarIOTwitterNar; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; @@ -41,31 +34,17 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.function.Consumer; import javax.ws.rs.core.Response; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.apache.pulsar.broker.authentication.AuthenticationParameters; -import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.Namespaces; -import org.apache.pulsar.client.admin.Packages; -import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; -import org.apache.pulsar.client.admin.Tenants; import org.apache.pulsar.common.functions.UpdateOptionsImpl; import org.apache.pulsar.common.functions.Utils; import org.apache.pulsar.common.io.SourceConfig; -import org.apache.pulsar.common.nar.NarClassLoader; -import org.apache.pulsar.common.nar.NarClassLoaderBuilder; -import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.RestException; import org.apache.pulsar.functions.api.utils.IdentityFunction; @@ -75,159 +54,35 @@ import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; import org.apache.pulsar.functions.proto.Function.SinkSpec; import org.apache.pulsar.functions.proto.Function.SourceSpec; -import org.apache.pulsar.functions.runtime.RuntimeFactory; import org.apache.pulsar.functions.source.TopicSchema; -import org.apache.pulsar.functions.utils.FunctionCommon; import org.apache.pulsar.functions.utils.SourceConfigUtils; import org.apache.pulsar.functions.utils.io.ConnectorUtils; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.LeaderService; -import org.apache.pulsar.functions.worker.PulsarWorkerService; -import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; -import org.apache.pulsar.functions.worker.rest.api.PulsarFunctionTestTemporaryDirectory; import org.apache.pulsar.functions.worker.rest.api.SourcesImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.testng.annotations.AfterClass; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; /** * Unit test of {@link SourcesApiV3Resource}. */ -public class SourceApiV3ResourceTest { +public class SourceApiV3ResourceTest extends AbstractFunctionsResourceTest { - private static final String tenant = "test-tenant"; - private static final String namespace = "test-namespace"; private static final String source = "test-source"; private static final String outputTopic = "test-output-topic"; private static final String outputSerdeClassName = TopicSchema.DEFAULT_SERDE; private static final String TWITTER_FIRE_HOSE = "org.apache.pulsar.io.twitter.TwitterFireHose"; - private static final int parallelism = 1; - - private PulsarWorkerService mockedWorkerService; - private PulsarAdmin mockedPulsarAdmin; - private Tenants mockedTenants; - private Namespaces mockedNamespaces; - private Functions mockedFunctions; - private TenantInfoImpl mockedTenantInfo; - private List namespaceList = new LinkedList<>(); - private FunctionMetaDataManager mockedManager; - private FunctionRuntimeManager mockedFunctionRunTimeManager; - private RuntimeFactory mockedRuntimeFactory; - private Namespace mockedNamespace; private SourcesImpl resource; - private InputStream mockedInputStream; - private FormDataContentDisposition mockedFormData; - private FunctionMetaData mockedFunctionMetaData; - private LeaderService mockedLeaderService; - private Packages mockedPackages; - private PulsarFunctionTestTemporaryDirectory tempDirectory; - - private static NarClassLoader narClassLoader; - private static Map mockStaticContexts = new HashMap<>(); - - @BeforeClass - public void setupNarClassLoader() throws IOException { - narClassLoader = NarClassLoaderBuilder.builder().narFile(getPulsarIOTwitterNar()).build(); - } - - @AfterClass(alwaysRun = true) - public void cleanupNarClassLoader() throws IOException { - if (narClassLoader != null) { - narClassLoader.close(); - narClassLoader = null; - } - } - - @BeforeMethod - public void setup() throws Exception { - this.mockedManager = mock(FunctionMetaDataManager.class); - this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); - this.mockedRuntimeFactory = mock(RuntimeFactory.class); - this.mockedInputStream = mock(InputStream.class); - this.mockedNamespace = mock(Namespace.class); - this.mockedFormData = mock(FormDataContentDisposition.class); - when(mockedFormData.getFileName()).thenReturn("test"); - this.mockedTenantInfo = mock(TenantInfoImpl.class); - this.mockedPulsarAdmin = mock(PulsarAdmin.class); - this.mockedTenants = mock(Tenants.class); - this.mockedNamespaces = mock(Namespaces.class); - this.mockedFunctions = mock(Functions.class); - this.mockedLeaderService = mock(LeaderService.class); - this.mockedPackages = mock(Packages.class); - namespaceList.add(tenant + "/" + namespace); - - this.mockedWorkerService = mock(PulsarWorkerService.class); - when(mockedWorkerService.getFunctionMetaDataManager()).thenReturn(mockedManager); - when(mockedWorkerService.getLeaderService()).thenReturn(mockedLeaderService); - when(mockedWorkerService.getFunctionRuntimeManager()).thenReturn(mockedFunctionRunTimeManager); - when(mockedFunctionRunTimeManager.getRuntimeFactory()).thenReturn(mockedRuntimeFactory); - when(mockedWorkerService.getDlogNamespace()).thenReturn(mockedNamespace); - when(mockedWorkerService.isInitialized()).thenReturn(true); - when(mockedWorkerService.getBrokerAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedWorkerService.getFunctionAdmin()).thenReturn(mockedPulsarAdmin); - when(mockedPulsarAdmin.tenants()).thenReturn(mockedTenants); - when(mockedPulsarAdmin.namespaces()).thenReturn(mockedNamespaces); - when(mockedPulsarAdmin.functions()).thenReturn(mockedFunctions); - when(mockedPulsarAdmin.packages()).thenReturn(mockedPackages); - when(mockedTenants.getTenantInfo(any())).thenReturn(mockedTenantInfo); - when(mockedNamespaces.getNamespaces(any())).thenReturn(namespaceList); - when(mockedLeaderService.isLeader()).thenReturn(true); - doAnswer(invocationOnMock -> { - Files.copy(getPulsarIOTwitterNar().toPath(), Paths.get(invocationOnMock.getArgument(1, String.class)), - StandardCopyOption.REPLACE_EXISTING); - return null; - }).when(mockedPackages).download(any(), any()); - - // worker config - WorkerConfig workerConfig = new WorkerConfig() - .setWorkerId("test") - .setWorkerPort(8080) - .setFunctionMetadataTopicName("pulsar/functions") - .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); - tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); - tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); - when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); + @Override + protected void doSetup() { this.resource = spy(new SourcesImpl(() -> mockedWorkerService)); } - private void mockStatic(Class classStatic, Consumer> consumer) { - final MockedStatic mockedStatic = - mockStaticContexts.computeIfAbsent(classStatic.getName(), name -> Mockito.mockStatic(classStatic)); - consumer.accept(mockedStatic); - } - - @AfterMethod(alwaysRun = true) - public void cleanup() { - if (tempDirectory != null) { - tempDirectory.delete(); - } - mockStaticContexts.values().forEach(MockedStatic::close); - mockStaticContexts.clear(); - } - - private void mockWorkerUtils() { - mockStatic(WorkerUtils.class, - ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - }); - } - - private void mockWorkerUtils(Consumer> consumer) { - mockStatic(WorkerUtils.class, ctx -> { - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); - if (consumer != null) { - consumer.accept(ctx); - } - }); + @Override + protected FunctionDetails.ComponentType getComponentType() { + return FunctionDetails.ComponentType.SOURCE; } // @@ -297,8 +152,8 @@ public void testRegisterSourceMissingSourceName() { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source class UnknownClass must" - + " be in class path") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source class UnknownClass not " + + "found in class loader") public void testRegisterSourceWrongClassName() { try { testRegisterSourceMissingArguments( @@ -361,10 +216,8 @@ public void testRegisterSourceMissingPackageDetails() throws IOException { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source package does not have the" - + " correct format. Pulsar cannot determine if the package is a NAR package" - + " or JAR package. Source classname is not provided and attempts to load it as a NAR package " - + "produced the following error.") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source package doesn't contain" + + " the META-INF/services/pulsar-io.yaml file.") public void testRegisterSourceMissingPackageDetailsAndClassname() { try { testRegisterSourceMissingArguments( @@ -385,8 +238,8 @@ public void testRegisterSourceMissingPackageDetailsAndClassname() { } } - @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Failed to extract source class" - + " from archive") + @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source package doesn't contain" + + " the META-INF/services/pulsar-io.yaml file.") public void testRegisterSourceInvalidJarWithNoSource() throws IOException { try (InputStream inputStream = new FileInputStream(getPulsarIOInvalidNar())) { testRegisterSourceMissingArguments( @@ -524,7 +377,7 @@ public void testUpdateMissingSinkConfig() { } private void registerDefaultSource() throws IOException { - registerDefaultSourceWithPackageUrl("source://public/default/test@v1"); + registerDefaultSourceWithPackageUrl(getPulsarIOTwitterNar().toURI().toString()); } private void registerDefaultSourceWithPackageUrl(String packageUrl) throws IOException { @@ -565,8 +418,6 @@ public void testRegisterSourceUploadFailure() throws Exception { any(File.class), any(Namespace.class))) .thenThrow(new IOException("upload failure")); - - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(false); @@ -593,7 +444,7 @@ public void testRegisterSourceSuccess() throws Exception { @Test(timeOut = 20000) public void testRegisterSourceSuccessWithPackageName() throws IOException { - registerDefaultSourceWithPackageUrl("source://public/default/test@v1"); + registerDefaultSource(); } @Test(timeOut = 20000) @@ -621,14 +472,7 @@ public void testRegisterSourceConflictingFields() throws Exception { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); when(mockedManager.containsFunction(eq(actualTenant), eq(actualNamespace), eq(actualName))).thenReturn(false); - SourceConfig sourceConfig = new SourceConfig(); - sourceConfig.setTenant(tenant); - sourceConfig.setNamespace(namespace); - sourceConfig.setName(source); - sourceConfig.setClassName(TWITTER_FIRE_HOSE); - sourceConfig.setParallelism(parallelism); - sourceConfig.setTopicName(outputTopic); - sourceConfig.setSerdeClassName(outputSerdeClassName); + SourceConfig sourceConfig = createDefaultSourceConfig(); try (InputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { resource.registerSource( actualTenant, @@ -815,17 +659,19 @@ public void testUpdateSourceChangedParallelism() throws Exception { try { mockWorkerUtils(); - testUpdateSourceMissingArguments( - tenant, - namespace, - source, - null, - mockedFormData, - outputTopic, - outputSerdeClassName, - TWITTER_FIRE_HOSE, - parallelism + 1, - null); + try(FileInputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { + testUpdateSourceMissingArguments( + tenant, + namespace, + source, + inputStream, + mockedFormData, + outputTopic, + outputSerdeClassName, + TWITTER_FIRE_HOSE, + parallelism + 1, + null); + } } catch (RestException re) { assertEquals(re.getResponse().getStatusInfo(), Response.Status.BAD_REQUEST); throw re; @@ -836,31 +682,29 @@ public void testUpdateSourceChangedParallelism() throws Exception { public void testUpdateSourceChangedTopic() throws Exception { mockWorkerUtils(); - testUpdateSourceMissingArguments( - tenant, - namespace, - source, - null, - mockedFormData, - "DifferentTopic", - outputSerdeClassName, - TWITTER_FIRE_HOSE, - parallelism, - null); + try(FileInputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { + testUpdateSourceMissingArguments( + tenant, + namespace, + source, + inputStream, + mockedFormData, + "DifferentTopic", + outputSerdeClassName, + TWITTER_FIRE_HOSE, + parallelism, + null); + } } @Test - public void testUpdateSourceWithNoChange() { + public void testUpdateSourceWithNoChange() throws IOException { mockWorkerUtils(); // No change on config, SourceConfig sourceConfig = createDefaultSourceConfig(); mockStatic(SourceConfigUtils.class, ctx -> { ctx.when(() -> SourceConfigUtils.convertFromDetails(any())).thenReturn(sourceConfig); - ctx.when(() -> SourceConfigUtils.convert(any(), any())).thenCallRealMethod(); - ctx.when(() -> SourceConfigUtils.validateUpdate(any(), any())).thenCallRealMethod(); - ctx.when(() -> SourceConfigUtils.clone(any())).thenCallRealMethod(); - ctx.when(() -> SourceConfigUtils.validateAndExtractDetails(any(),any(),anyBoolean())).thenCallRealMethod(); }); mockFunctionCommon(sourceConfig.getTenant(), sourceConfig.getNamespace(), sourceConfig.getName()); @@ -903,16 +747,18 @@ public void testUpdateSourceWithNoChange() { // no changes but set the auth-update flag to true, should not fail UpdateOptionsImpl updateOptions = new UpdateOptionsImpl(); updateOptions.setUpdateAuthData(true); - resource.updateSource( - sourceConfig.getTenant(), - sourceConfig.getNamespace(), - sourceConfig.getName(), - null, - mockedFormData, - null, - sourceConfig, - null, - updateOptions); + try (InputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { + resource.updateSource( + sourceConfig.getTenant(), + sourceConfig.getNamespace(), + sourceConfig.getName(), + inputStream, + mockedFormData, + null, + sourceConfig, + null, + updateOptions); + } } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source parallelism must be a " @@ -997,14 +843,6 @@ private void mockFunctionCommon(String tenant, String namespace, String function }); mockStatic(ClassLoaderUtils.class, c -> { }); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) - .thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) - .thenReturn(narClassLoader); - }); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); @@ -1014,49 +852,25 @@ private void mockFunctionCommon(String tenant, String namespace, String function } private void updateDefaultSource() throws Exception { - updateDefaultSourceWithPackageUrl(null); + updateDefaultSourceWithPackageUrl(getPulsarIOTwitterNar().toURI().toString()); } private void updateDefaultSourceWithPackageUrl(String packageUrl) throws Exception { - SourceConfig sourceConfig = new SourceConfig(); - sourceConfig.setTenant(tenant); - sourceConfig.setNamespace(namespace); - sourceConfig.setName(source); - sourceConfig.setClassName(TWITTER_FIRE_HOSE); - sourceConfig.setParallelism(parallelism); - sourceConfig.setTopicName(outputTopic); - sourceConfig.setSerdeClassName(outputSerdeClassName); - - mockStatic(ConnectorUtils.class, c -> { - }); - - mockStatic(ClassLoaderUtils.class, c -> { - }); - - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.createPkgTempFile()).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) - .thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) - .thenReturn(narClassLoader); - }); + SourceConfig sourceConfig = createDefaultSourceConfig(); this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); - try (InputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { - resource.updateSource( - tenant, - namespace, - source, - inputStream, - mockedFormData, - packageUrl, - sourceConfig, - null, null); - } + resource.updateSource( + tenant, + namespace, + source, + null, + mockedFormData, + packageUrl, + sourceConfig, + null, null); } @Test(expectedExceptions = RestException.class, expectedExceptionsMessageRegExp = "Source test-source doesn't " + @@ -1079,11 +893,25 @@ public void testUpdateSourceUploadFailure() throws Exception { anyString(), any(File.class), any(Namespace.class))).thenThrow(new IOException("upload failure")); - ctx.when(() -> WorkerUtils.dumpToTmpFile(any())).thenCallRealMethod(); }); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); - updateDefaultSource(); + SourceConfig sourceConfig = createDefaultSourceConfig(); + this.mockedFunctionMetaData = + FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); + when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); + + try(InputStream inputStream = new FileInputStream(getPulsarIOTwitterNar())) { + resource.updateSource( + tenant, + namespace, + source, + inputStream, + mockedFormData, + null, + sourceConfig, + null, null); + } } catch (RestException re) { assertEquals(re.getResponse().getStatusInfo(), Response.Status.INTERNAL_SERVER_ERROR); throw re; @@ -1103,16 +931,9 @@ public void testUpdateSourceSuccess() throws Exception { public void testUpdateSourceWithUrl() throws Exception { Configurator.setRootLevel(Level.DEBUG); - String filePackageUrl = getPulsarIOCassandraNar().toURI().toString(); + String filePackageUrl = getPulsarIOTwitterNar().toURI().toString(); - SourceConfig sourceConfig = new SourceConfig(); - sourceConfig.setTopicName(outputTopic); - sourceConfig.setSerdeClassName(outputSerdeClassName); - sourceConfig.setTenant(tenant); - sourceConfig.setNamespace(namespace); - sourceConfig.setName(source); - sourceConfig.setClassName(TWITTER_FIRE_HOSE); - sourceConfig.setParallelism(parallelism); + SourceConfig sourceConfig = createDefaultSourceConfig(); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(source))).thenReturn(true); mockStatic(ConnectorUtils.class, c -> { @@ -1120,15 +941,6 @@ public void testUpdateSourceWithUrl() throws Exception { mockStatic(ClassLoaderUtils.class, c -> { }); - mockStatic(FunctionCommon.class, ctx -> { - ctx.when(() -> FunctionCommon.extractFileFromPkgURL(any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); - ctx.when(() -> FunctionCommon.getSourceType(argThat(clazz -> clazz.getName().equals(TWITTER_FIRE_HOSE)))) - .thenReturn(String.class); - ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())) - .thenReturn(narClassLoader); - }); - this.mockedFunctionMetaData = FunctionMetaData.newBuilder().setFunctionDetails(createDefaultFunctionDetails()).build(); when(mockedManager.getFunctionMetaData(any(), any(), any())).thenReturn(mockedFunctionMetaData); diff --git a/tests/docker-images/latest-version-image/conf/functions_worker.conf b/tests/docker-images/latest-version-image/conf/functions_worker.conf index 8072639a0d4a2..6feb660231cec 100644 --- a/tests/docker-images/latest-version-image/conf/functions_worker.conf +++ b/tests/docker-images/latest-version-image/conf/functions_worker.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/functions_worker.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/pulsar/logs/functions",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar functions-worker user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java index ce308e4a386d5..c648f6968e24c 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java @@ -525,37 +525,18 @@ public synchronized void setupFunctionWorkers(String suffix, FunctionRuntimeType } private void startFunctionWorkersWithProcessContainerFactory(String suffix, int numFunctionWorkers) { - String serviceUrl = "pulsar://pulsar-broker-0:" + PulsarContainer.BROKER_PORT; - String httpServiceUrl = "http://pulsar-broker-0:" + PulsarContainer.BROKER_HTTP_PORT; workerContainers.putAll(runNumContainers( "functions-worker-process-" + suffix, numFunctionWorkers, - (name) -> new WorkerContainer(clusterName, name) - .withNetwork(network) - .withNetworkAliases(name) - // worker settings - .withEnv("PF_workerId", name) - .withEnv("PF_workerHostname", name) - .withEnv("PF_workerPort", "" + PulsarContainer.BROKER_HTTP_PORT) - .withEnv("PF_pulsarFunctionsCluster", clusterName) - .withEnv("PF_pulsarServiceUrl", serviceUrl) - .withEnv("PF_pulsarWebServiceUrl", httpServiceUrl) - // script - .withEnv("clusterName", clusterName) - .withEnv("zookeeperServers", ZKContainer.NAME) - // bookkeeper tools - .withEnv("zkServers", ZKContainer.NAME) + (name) -> createWorkerContainer(name) )); this.startWorkers(); } - private void startFunctionWorkersWithThreadContainerFactory(String suffix, int numFunctionWorkers) { + private WorkerContainer createWorkerContainer(String name) { String serviceUrl = "pulsar://pulsar-broker-0:" + PulsarContainer.BROKER_PORT; String httpServiceUrl = "http://pulsar-broker-0:" + PulsarContainer.BROKER_HTTP_PORT; - workerContainers.putAll(runNumContainers( - "functions-worker-thread-" + suffix, - numFunctionWorkers, - (name) -> new WorkerContainer(clusterName, name) + return new WorkerContainer(clusterName, name) .withNetwork(network) .withNetworkAliases(name) // worker settings @@ -565,13 +546,21 @@ private void startFunctionWorkersWithThreadContainerFactory(String suffix, int n .withEnv("PF_pulsarFunctionsCluster", clusterName) .withEnv("PF_pulsarServiceUrl", serviceUrl) .withEnv("PF_pulsarWebServiceUrl", httpServiceUrl) - .withEnv("PF_functionRuntimeFactoryClassName", "org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory") - .withEnv("PF_functionRuntimeFactoryConfigs_threadGroupName", "pf-container-group") // script .withEnv("clusterName", clusterName) .withEnv("zookeeperServers", ZKContainer.NAME) // bookkeeper tools - .withEnv("zkServers", ZKContainer.NAME) + .withEnv("zkServers", ZKContainer.NAME); + } + + private void startFunctionWorkersWithThreadContainerFactory(String suffix, int numFunctionWorkers) { + workerContainers.putAll(runNumContainers( + "functions-worker-thread-" + suffix, + numFunctionWorkers, + (name) -> createWorkerContainer(name) + .withEnv("PF_functionRuntimeFactoryClassName", + "org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory") + .withEnv("PF_functionRuntimeFactoryConfigs_threadGroupName", "pf-container-group") )); this.startWorkers(); } From e3f51157340658ce6cc1f7c0f78ed1e1e50a130b Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 26 Feb 2024 17:06:08 +0200 Subject: [PATCH 470/494] Fix byte-buddy version in presto LICENSE --- pulsar-sql/presto-distribution/LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index f25ca8ec78bda..9e1beb639f4c9 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -300,7 +300,7 @@ The Apache Software License, Version 2.0 - jetty-util-9.4.53.v20231009.jar - jetty-util-ajax-9.4.53.v20231009.jar * Byte Buddy - - byte-buddy-1.11.13.jar + - byte-buddy-1.14.12.jar * Apache BVal - bval-jsr-2.0.5.jar * Bytecode From 1a0a457d5608a153e681125fea710a7907804fc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 16:58:41 -0800 Subject: [PATCH 471/494] [fix] Bump org.apache.solr:solr-core from 8.11.1 to 8.11.3 in /pulsar-io/solr (#22047) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 7a90426253e96a995e5d3a254c76cb80a3d54c7b) --- pulsar-io/solr/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index ffd1340f5f3c2..4ba68befb0390 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -29,7 +29,7 @@ - 8.11.1 + 8.11.3 pulsar-io-solr From 860f085ceaebe64fe5de3a6dfb90749709635207 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Wed, 27 Dec 2023 10:07:49 +0800 Subject: [PATCH 472/494] [improve][ci] Exclude jose4j to avoid CVE-2023-31582 (#21791) (cherry picked from commit d9c55b4f5c70fa13185fdf0290d4513b0380f216) --- pom.xml | 4 ++++ pulsar-io/debezium/core/pom.xml | 4 ++++ pulsar-io/kafka-connect-adaptor/pom.xml | 22 ++++++++++++++++++++++ pulsar-io/kafka/pom.xml | 6 ++++++ pulsar-io/solr/pom.xml | 6 ++++++ 5 files changed, 42 insertions(+) diff --git a/pom.xml b/pom.xml index 6e4b31aa28957..122b551d0e818 100644 --- a/pom.xml +++ b/pom.xml @@ -550,6 +550,10 @@ flexible messaging model and an intuitive client API. com.squareup.okio okio + + jose4j + org.bitbucket.b_c + diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index 5e08eb42b1767..b358ef2e9f0f9 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -71,6 +71,10 @@ org.apache.kafka kafka-log4j-appender + + jose4j + org.bitbucket.b_c + diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index 51cd57cb13ad1..e820b17568fba 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -73,6 +73,10 @@ org.eclipse.jetty * + + jose4j + org.bitbucket.b_c + @@ -80,12 +84,24 @@ org.apache.kafka connect-json ${kafka-client.version} + + + jose4j + org.bitbucket.b_c + + org.apache.kafka connect-api ${kafka-client.version} + + + jose4j + org.bitbucket.b_c + + @@ -136,6 +152,12 @@ connect-file ${kafka-client.version} test + + + jose4j + org.bitbucket.b_c + + diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index adbbce1348e39..290e03c7b340e 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -84,6 +84,12 @@ org.apache.kafka kafka-clients ${kafka-client.version} + + + jose4j + org.bitbucket.b_c + + diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 4ba68befb0390..9d2f7410a92fd 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -65,6 +65,12 @@ org.apache.solr solr-core ${solr.version} + + + jose4j + org.bitbucket.b_c + + test From e2ffea16b5d89ada8e043b14a1d55e2d778aeaac Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 27 Feb 2024 00:03:16 +0800 Subject: [PATCH 473/494] [fix][test] Fix test testTransactionBufferMetrics (#22117) (cherry picked from commit 0fc9f4465288d1f9938ea717ea2e7c8ff02ebb60) --- .../broker/transaction/buffer/TransactionBufferClientTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java index 3873d9d37b20b..1dc086dbe3470 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionBufferClientTest.java @@ -210,6 +210,8 @@ public void testAbortOnSubscription() throws ExecutionException, InterruptedExce @Test public void testTransactionBufferMetrics() throws Exception { + this.cleanup(); + this.setup(); //Test commit for (int i = 0; i < partitions; i++) { String topic = partitionedTopicName.getPartition(i).toString(); From 6e988ef347ffdf31e77bc20882d84c4b6eb74adc Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Fri, 23 Feb 2024 20:23:38 +0800 Subject: [PATCH 474/494] [improve][admin] Expose the offload threshold in seconds to the amdin (#22101) (cherry picked from commit 48c7e322fecfbcee0df758bdaf7e9b4263f2835e) --- .../org/apache/pulsar/admin/cli/CmdNamespaces.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 998591f8177d1..28e8fb8d5f5b2 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1962,7 +1962,8 @@ private class GetOffloadThreshold extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - print(getAdmin().namespaces().getOffloadThreshold(namespace)); + print("offloadThresholdInBytes: " + getAdmin().namespaces().getOffloadThreshold(namespace)); + print("offloadThresholdInSeconds: " + getAdmin().namespaces().getOffloadThresholdInSeconds(namespace)); } } @@ -1980,11 +1981,18 @@ private class SetOffloadThreshold extends CliCommand { required = true) private String thresholdStr = "-1"; + @Parameter(names = {"--time", "-t"}, + description = "Maximum number of seconds stored on the pulsar cluster for a topic" + + " before the broker will start offloading to longterm storage (eg: 10m, 5h, 3d, 2w).", + converter = TimeUnitToSecondsConverter.class) + private Long thresholdInSeconds = -1L; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); long threshold = validateSizeString(thresholdStr); getAdmin().namespaces().setOffloadThreshold(namespace, threshold); + getAdmin().namespaces().setOffloadThresholdInSeconds(namespace, thresholdInSeconds); } } From 8ea425e0cf1602cb9a84b88cc02f8a443e5f24cd Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 23 Feb 2024 23:13:08 +0800 Subject: [PATCH 475/494] [improve] [broker] Do not try to open ML when the topic meta does not exist and do not expect to create a new one. #21995 (#22004) Co-authored-by: Jiwe Guo (cherry picked from commit 1c652f5519e013340e08950fc9705da4e54bf22a) --- .../pulsar/broker/service/BrokerService.java | 80 ++++++++++--------- .../broker/TopicEventsListenerTest.java | 33 ++++---- .../pulsar/broker/admin/AdminApi2Test.java | 28 +++++++ .../broker/admin/TopicAutoCreationTest.java | 14 ++-- .../persistent/PersistentTopicTest.java | 3 +- 5 files changed, 99 insertions(+), 59 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index c5775e023bbab..8997e1e98ee63 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1051,43 +1051,49 @@ public CompletableFuture> getTopic(final TopicName topicName, bo } final boolean isPersistentTopic = topicName.getDomain().equals(TopicDomain.persistent); if (isPersistentTopic) { - final CompletableFuture> topicPoliciesFuture = - getTopicPoliciesBypassSystemTopic(topicName); - return topicPoliciesFuture.exceptionally(ex -> { - final Throwable rc = FutureUtil.unwrapCompletionException(ex); - final String errorInfo = String.format("Topic creation encountered an exception by initialize" - + " topic policies service. topic_name=%s error_message=%s", topicName, rc.getMessage()); - log.error(errorInfo, rc); - throw FutureUtil.wrapToCompletionException(new ServiceUnitNotReadyException(errorInfo)); - }).thenCompose(optionalTopicPolicies -> { - final TopicPolicies topicPolicies = optionalTopicPolicies.orElse(null); - return topics.computeIfAbsent(topicName.toString(), (tpName) -> { - if (topicName.isPartitioned()) { - final TopicName topicNameEntity = TopicName.get(topicName.getPartitionedTopicName()); - return fetchPartitionedTopicMetadataAsync(topicNameEntity) - .thenCompose((metadata) -> { - // Allow crate non-partitioned persistent topic that name includes `partition` - if (metadata.partitions == 0 - || topicName.getPartitionIndex() < metadata.partitions) { - return loadOrCreatePersistentTopic(tpName, createIfMissing, - properties, topicPolicies); - } - final String errorMsg = - String.format("Illegal topic partition name %s with max allowed " - + "%d partitions", topicName, metadata.partitions); - log.warn(errorMsg); - return FutureUtil - .failedFuture(new BrokerServiceException.NotAllowedException(errorMsg)); - }); - } - return loadOrCreatePersistentTopic(tpName, createIfMissing, properties, topicPolicies); - }).thenCompose(optionalTopic -> { - if (!optionalTopic.isPresent() && createIfMissing) { - log.warn("[{}] Try to recreate the topic with createIfMissing=true " - + "but the returned topic is empty", topicName); - return getTopic(topicName, createIfMissing, properties); - } - return CompletableFuture.completedFuture(optionalTopic); + return pulsar.getPulsarResources().getTopicResources().persistentTopicExists(topicName) + .thenCompose(exists -> { + if (!exists && !createIfMissing) { + return CompletableFuture.completedFuture(Optional.empty()); + } + return getTopicPoliciesBypassSystemTopic(topicName).exceptionally(ex -> { + final Throwable rc = FutureUtil.unwrapCompletionException(ex); + final String errorInfo = String.format("Topic creation encountered an exception by initialize" + + " topic policies service. topic_name=%s error_message=%s", topicName, + rc.getMessage()); + log.error(errorInfo, rc); + throw FutureUtil.wrapToCompletionException(new ServiceUnitNotReadyException(errorInfo)); + }).thenCompose(optionalTopicPolicies -> { + final TopicPolicies topicPolicies = optionalTopicPolicies.orElse(null); + return topics.computeIfAbsent(topicName.toString(), (tpName) -> { + if (topicName.isPartitioned()) { + final TopicName topicNameEntity = TopicName.get(topicName.getPartitionedTopicName()); + return fetchPartitionedTopicMetadataAsync(topicNameEntity) + .thenCompose((metadata) -> { + // Allow crate non-partitioned persistent topic that name includes + // `partition` + if (metadata.partitions == 0 + || topicName.getPartitionIndex() < metadata.partitions) { + return loadOrCreatePersistentTopic(tpName, createIfMissing, + properties, topicPolicies); + } + final String errorMsg = + String.format("Illegal topic partition name %s with max allowed " + + "%d partitions", topicName, metadata.partitions); + log.warn(errorMsg); + return FutureUtil.failedFuture( + new BrokerServiceException.NotAllowedException(errorMsg)); + }); + } + return loadOrCreatePersistentTopic(tpName, createIfMissing, properties, topicPolicies); + }).thenCompose(optionalTopic -> { + if (!optionalTopic.isPresent() && createIfMissing) { + log.warn("[{}] Try to recreate the topic with createIfMissing=true " + + "but the returned topic is empty", topicName); + return getTopic(topicName, createIfMissing, properties); + } + return CompletableFuture.completedFuture(optionalTopic); + }); }); }); } else { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java index e6459bbf74c31..ceb3c1d0d9335 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/TopicEventsListenerTest.java @@ -126,7 +126,7 @@ public void testEvents(String topicTypePersistence, String topicTypePartitioned, boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); events.clear(); if (topicTypePartitioned.equals("partitioned")) { @@ -150,7 +150,7 @@ public void testEventsWithUnload(String topicTypePersistence, String topicTypePa boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); events.clear(); admin.topics().unload(topicName); @@ -182,7 +182,7 @@ public void testEventsActiveSub(String topicTypePersistence, String topicTypePar boolean forceDelete) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); Consumer consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub").subscribe(); Producer producer = pulsarClient.newProducer().topic(topicName).create(); @@ -238,7 +238,7 @@ public void testEventsActiveSub(String topicTypePersistence, String topicTypePar public void testTopicAutoGC(String topicTypePersistence, String topicTypePartitioned) throws Exception { String topicName = topicTypePersistence + "://" + namespace + "/" + "topic-" + UUID.randomUUID(); - createTopicAndVerifyEvents(topicTypePartitioned, topicName); + createTopicAndVerifyEvents(topicTypePersistence, topicTypePartitioned, topicName); admin.namespaces().setInactiveTopicPolicies(namespace, new InactiveTopicPolicies(InactiveTopicDeleteMode.delete_when_no_subscriptions, 1, true)); @@ -262,25 +262,21 @@ public void testTopicAutoGC(String topicTypePersistence, String topicTypePartiti ); } - private void createTopicAndVerifyEvents(String topicTypePartitioned, String topicName) throws Exception { + private void createTopicAndVerifyEvents(String topicDomain, String topicTypePartitioned, String topicName) throws Exception { final String[] expectedEvents; - if (topicTypePartitioned.equals("partitioned")) { - topicNameToWatch = topicName + "-partition-1"; - admin.topics().createPartitionedTopic(topicName, 2); - triggerPartitionsCreation(topicName); - + if (topicDomain.equalsIgnoreCase("persistent") || topicTypePartitioned.equals("partitioned")) { expectedEvents = new String[]{ "LOAD__BEFORE", "CREATE__BEFORE", "CREATE__SUCCESS", "LOAD__SUCCESS" }; - } else { - topicNameToWatch = topicName; - admin.topics().createNonPartitionedTopic(topicName); - expectedEvents = new String[]{ + // Before https://github.com/apache/pulsar/pull/21995, Pulsar will skip create topic if the topic + // was already exists, and the action "check topic exists" will try to load Managed ledger, + // the check triggers two exrtra events: [LOAD__BEFORE, LOAD__FAILURE]. + // #21995 fixed this wrong behavior, so remove these two events. "LOAD__BEFORE", "LOAD__FAILURE", "LOAD__BEFORE", @@ -288,7 +284,14 @@ private void createTopicAndVerifyEvents(String topicTypePartitioned, String topi "CREATE__SUCCESS", "LOAD__SUCCESS" }; - + } + if (topicTypePartitioned.equals("partitioned")) { + topicNameToWatch = topicName + "-partition-1"; + admin.topics().createPartitionedTopic(topicName, 2); + triggerPartitionsCreation(topicName); + } else { + topicNameToWatch = topicName; + admin.topics().createNonPartitionedTopic(topicName); } Awaitility.waitAtMost(10, TimeUnit.SECONDS).untilAsserted(() -> diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index 11564cdf72188..fe3c8b591ed6c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -3417,4 +3417,32 @@ private void testAnalyzeSubscriptionBacklogNotCauseStuck() throws Exception { producer.close(); admin.topics().delete(topic); } + + @Test + public void testGetStatsIfPartitionNotExists() throws Exception { + // create topic. + final String partitionedTp = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp"); + admin.topics().createPartitionedTopic(partitionedTp, 1); + TopicName partition0 = TopicName.get(partitionedTp).getPartition(0); + boolean topicExists1 = pulsar.getBrokerService().getTopic(partition0.toString(), false).join().isPresent(); + assertTrue(topicExists1); + // Verify topics-stats works. + TopicStats topicStats = admin.topics().getStats(partition0.toString()); + assertNotNull(topicStats); + + // Delete partition and call topic-stats again. + admin.topics().delete(partition0.toString()); + boolean topicExists2 = pulsar.getBrokerService().getTopic(partition0.toString(), false).join().isPresent(); + assertFalse(topicExists2); + // Verify: respond 404. + try { + admin.topics().getStats(partition0.toString()); + fail("Should respond 404 after the partition was deleted"); + } catch (Exception ex) { + assertTrue(ex.getMessage().contains("Topic partitions were not yet created")); + } + + // cleanup. + admin.topics().deletePartitionedTopic(partitionedTp); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java index c9138beee52d1..a75ae78cef393 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicAutoCreationTest.java @@ -149,10 +149,11 @@ public void testPartitionedTopicAutoCreationForbiddenDuringNamespaceDeletion() .sendTimeout(1, TimeUnit.SECONDS) .topic(topic) .create()) { - } catch (PulsarClientException.LookupException expected) { - String msg = "Namespace bundle for topic (%s) not served by this instance"; + } catch (PulsarClientException.TopicDoesNotExistException expected) { + // Since the "policies.deleted" is "true", the value of "isAllowAutoTopicCreationAsync" will be false, + // so the "TopicDoesNotExistException" is expected. log.info("Expected error", expected); - assertTrue(expected.getMessage().contains(String.format(msg, topic)) + assertTrue(expected.getMessage().contains(topic) || expected.getMessage().contains(topicPoliciesServiceInitException)); } @@ -160,10 +161,11 @@ public void testPartitionedTopicAutoCreationForbiddenDuringNamespaceDeletion() .topic(topic) .subscriptionName("test") .subscribe()) { - } catch (PulsarClientException.LookupException expected) { - String msg = "Namespace bundle for topic (%s) not served by this instance"; + } catch (PulsarClientException.TopicDoesNotExistException expected) { + // Since the "policies.deleted" is "true", the value of "isAllowAutoTopicCreationAsync" will be false, + // so the "TopicDoesNotExistException" is expected. log.info("Expected error", expected); - assertTrue(expected.getMessage().contains(String.format(msg, topic)) + assertTrue(expected.getMessage().contains(topic) || expected.getMessage().contains(topicPoliciesServiceInitException)); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java index 4f3e1e930bd51..4b4aa5b45d31a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/PersistentTopicTest.java @@ -295,7 +295,8 @@ public void testPersistentPartitionedTopicUnload() throws Exception { assertFalse(pulsar.getBrokerService().getTopics().containsKey(topicName)); pulsar.getBrokerService().getTopicIfExists(topicName).get(); - assertTrue(pulsar.getBrokerService().getTopics().containsKey(topicName)); + // The map topics should only contain partitions, does not contain partitioned topic. + assertFalse(pulsar.getBrokerService().getTopics().containsKey(topicName)); // ref of partitioned-topic name should be empty assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent()); From 2a050d17c9b87d8a4868a5613a5d2567126dbdb1 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Sat, 24 Feb 2024 08:04:37 +0800 Subject: [PATCH 476/494] [improve][broker] Cache the internal writer when sent to system topic. (#22099) (cherry picked from commit 8607905989081421c452e87db7b1eedececd7977) --- .../SystemTopicBasedTopicPoliciesService.java | 84 ++++++++++++------- .../TopicPoliciesSystemTopicClient.java | 10 ++- ...temTopicBasedTopicPoliciesServiceTest.java | 35 ++++++++ 3 files changed, 97 insertions(+), 32 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index 80fecbe67b646..71f78e21f938f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -19,6 +19,8 @@ package org.apache.pulsar.broker.service; import static java.util.Objects.requireNonNull; +import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import java.util.HashSet; @@ -29,6 +31,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.MutablePair; @@ -84,10 +87,25 @@ public class SystemTopicBasedTopicPoliciesService implements TopicPoliciesServic @VisibleForTesting final Map>> listeners = new ConcurrentHashMap<>(); + private final AsyncLoadingCache> writerCaches; + public SystemTopicBasedTopicPoliciesService(PulsarService pulsarService) { this.pulsarService = pulsarService; this.clusterName = pulsarService.getConfiguration().getClusterName(); this.localCluster = Sets.newHashSet(clusterName); + this.writerCaches = Caffeine.newBuilder() + .expireAfterAccess(5, TimeUnit.MINUTES) + .removalListener((namespaceName, writer, cause) -> { + ((SystemTopicClient.Writer) writer).closeAsync().exceptionally(ex -> { + log.error("[{}] Close writer error.", namespaceName, ex); + return null; + }); + }) + .buildAsync((namespaceName, executor) -> { + SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory + .createTopicPoliciesSystemTopicClient(namespaceName); + return systemTopicClient.newWriterAsync(); + }); } @Override @@ -122,39 +140,32 @@ private CompletableFuture sendTopicPolicyEvent(TopicName topicName, Action } catch (PulsarServerException e) { return CompletableFuture.failedFuture(e); } - - SystemTopicClient systemTopicClient = namespaceEventsSystemTopicFactory - .createTopicPoliciesSystemTopicClient(topicName.getNamespaceObject()); - - return systemTopicClient.newWriterAsync() - .thenCompose(writer -> { - PulsarEvent event = getPulsarEvent(topicName, actionType, policies); - CompletableFuture writeFuture = - ActionType.DELETE.equals(actionType) ? writer.deleteAsync(getEventKey(event), event) - : writer.writeAsync(getEventKey(event), event); - return writeFuture.handle((messageId, e) -> { - if (e != null) { - return CompletableFuture.failedFuture(e); + CompletableFuture result = new CompletableFuture<>(); + writerCaches.get(topicName.getNamespaceObject()) + .whenComplete((writer, cause) -> { + if (cause != null) { + writerCaches.synchronous().invalidate(topicName.getNamespaceObject()); + result.completeExceptionally(cause); } else { - if (messageId != null) { - return CompletableFuture.completedFuture(null); - } else { - return CompletableFuture.failedFuture( - new RuntimeException("Got message id is null.")); - } - } - }).thenRun(() -> - writer.closeAsync().whenComplete((v, cause) -> { - if (cause != null) { - log.error("[{}] Close writer error.", topicName, cause); + PulsarEvent event = getPulsarEvent(topicName, actionType, policies); + CompletableFuture writeFuture = ActionType.DELETE.equals(actionType) + ? writer.deleteAsync(getEventKey(event), event) + : writer.writeAsync(getEventKey(event), event); + writeFuture.whenComplete((messageId, e) -> { + if (e != null) { + result.completeExceptionally(e); + } else { + if (messageId != null) { + result.complete(null); } else { - if (log.isDebugEnabled()) { - log.debug("[{}] Close writer success.", topicName); - } + result.completeExceptionally( + new RuntimeException("Got message id is null.")); } - }) - ); + } + }); + } }); + return result; }); } @@ -364,7 +375,7 @@ public CompletableFuture removeOwnedNamespaceBundleAsync(NamespaceBundle n } AtomicInteger bundlesCount = ownedBundlesCountPerNamespace.get(namespace); if (bundlesCount == null || bundlesCount.decrementAndGet() <= 0) { - cleanCacheAndCloseReader(namespace, true); + cleanCacheAndCloseReader(namespace, true, true); } return CompletableFuture.completedFuture(null); } @@ -440,6 +451,14 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp } private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount) { + cleanCacheAndCloseReader(namespace, cleanOwnedBundlesCount, false); + } + + private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount, + boolean cleanWriterCache) { + if (cleanWriterCache) { + writerCaches.synchronous().invalidate(namespace); + } CompletableFuture> readerFuture = readerCaches.remove(namespace); if (cleanOwnedBundlesCount) { @@ -688,5 +707,10 @@ protected Map>> getListeners( return listeners; } + @VisibleForTesting + protected AsyncLoadingCache> getWriterCaches() { + return writerCaches; + } + private static final Logger log = LoggerFactory.getLogger(SystemTopicBasedTopicPoliciesService.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java index 3fd8921c15efa..b7cff2e08c2d0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/systopic/TopicPoliciesSystemTopicClient.java @@ -30,6 +30,8 @@ import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TypedMessageBuilder; +import org.apache.pulsar.client.api.schema.SchemaDefinition; +import org.apache.pulsar.client.internal.DefaultImplementation; import org.apache.pulsar.common.events.ActionType; import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.TopicName; @@ -41,13 +43,17 @@ */ public class TopicPoliciesSystemTopicClient extends SystemTopicClientBase { + static Schema avroSchema = DefaultImplementation.getDefaultImplementation() + .newAvroSchema(SchemaDefinition.builder().withPojo(PulsarEvent.class).build()); + public TopicPoliciesSystemTopicClient(PulsarClient client, TopicName topicName) { super(client, topicName); + } @Override protected CompletableFuture> newWriterAsyncInternal() { - return client.newProducer(Schema.AVRO(PulsarEvent.class)) + return client.newProducer(avroSchema) .topic(topicName.toString()) .enableBatching(false) .createAsync() @@ -61,7 +67,7 @@ protected CompletableFuture> newWriterAsyncInternal() { @Override protected CompletableFuture> newReaderAsyncInternal() { - return client.newReader(Schema.AVRO(PulsarEvent.class)) + return client.newReader(avroSchema) .topic(topicName.toString()) .startMessageId(MessageId.earliest) .readCompacted(true) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index ba5e42867d31f..cde41ffae6835 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -33,6 +33,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -42,6 +43,7 @@ import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.impl.Backoff; import org.apache.pulsar.client.impl.BackoffBuilder; import org.apache.pulsar.common.events.PulsarEvent; @@ -66,6 +68,8 @@ public class SystemTopicBasedTopicPoliciesServiceTest extends MockedPulsarServic private static final String NAMESPACE2 = "system-topic/namespace-2"; private static final String NAMESPACE3 = "system-topic/namespace-3"; + private static final String NAMESPACE4 = "system-topic/namespace-4"; + private static final TopicName TOPIC1 = TopicName.get("persistent", NamespaceName.get(NAMESPACE1), "topic-1"); private static final TopicName TOPIC2 = TopicName.get("persistent", NamespaceName.get(NAMESPACE1), "topic-2"); private static final TopicName TOPIC3 = TopicName.get("persistent", NamespaceName.get(NAMESPACE2), "topic-1"); @@ -428,4 +432,35 @@ public void testGetTopicPoliciesWithCleanCache() throws Exception { result.join(); } + + @Test + public void testWriterCache() throws Exception { + admin.namespaces().createNamespace(NAMESPACE4); + for (int i = 1; i <= 5; i ++) { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + i; + admin.topics().createNonPartitionedTopic(topicName); + pulsarClient.newProducer(Schema.STRING).topic(topicName).create().close(); + } + @Cleanup("shutdown") + ExecutorService executorService = Executors.newFixedThreadPool(5); + for (int i = 1; i <= 5; i ++) { + int finalI = i; + executorService.execute(() -> { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + finalI; + try { + admin.topicPolicies().setMaxConsumers(topicName, 2); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + SystemTopicBasedTopicPoliciesService service = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + Assert.assertNotNull(service.getWriterCaches().synchronous().get(NamespaceName.get(NAMESPACE4))); + for (int i = 1; i <= 5; i ++) { + final String topicName = "persistent://" + NAMESPACE4 + "/testWriterCache" + i; + admin.topics().delete(topicName); + } + admin.namespaces().deleteNamespace(NAMESPACE4); + Assert.assertNull(service.getWriterCaches().synchronous().getIfPresent(NamespaceName.get(NAMESPACE4))); + } } From b961c4906f6992f298e73f7200f85bab5f148a7e Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 26 Feb 2024 11:06:01 +0800 Subject: [PATCH 477/494] [fix] [broker] Enabling batch causes negative unackedMessages due to ack and delivery concurrency (#22090) (cherry picked from commit 1b1cfb58f4e64c91a311950d002b0c755bd9604f) --- .../pulsar/broker/service/Consumer.java | 2 +- .../BatchMessageWithBatchIndexLevelTest.java | 182 ++++++++++++++++++ 2 files changed, 183 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 5c84ccf64fdde..d8ed99bb87436 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -327,7 +327,7 @@ public Future sendMessages(final List entries, EntryBatch if (pendingAcks != null) { int batchSize = batchSizes.getBatchSize(i); int stickyKeyHash = getStickyKeyHash(entry); - long[] ackSet = getCursorAckSet(PositionImpl.get(entry.getLedgerId(), entry.getEntryId())); + long[] ackSet = batchIndexesAcks == null ? null : batchIndexesAcks.getAckSet(i); if (ackSet != null) { unackedMessages -= (batchSize - BitSet.valueOf(ackSet).cardinality()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java index b2fbe824b3305..3a4cee7f2be83 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -18,8 +18,17 @@ */ package org.apache.pulsar.broker.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; +import com.carrotsearch.hppc.ObjectSet; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -28,10 +37,14 @@ import lombok.Cleanup; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl; +import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageIdAdv; @@ -39,8 +52,10 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; +import org.apache.pulsar.client.impl.BatchMessageIdImpl; import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.awaitility.Awaitility; import org.testng.Assert; import org.testng.annotations.BeforeClass; @@ -401,4 +416,171 @@ public void testMixIndexAndNonIndexUnAckMessageCount() throws Exception { assertEquals(admin.topics().getStats(topicName).getSubscriptions() .get("sub").getUnackedMessages(), 0); } + + @Test + public void testUnAckMessagesWhenConcurrentDeliveryAndAck() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/tp"); + final String subName = "s1"; + final int receiverQueueSize = 500; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics().createSubscription(topicName, subName, MessageId.earliest); + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .receiverQueueSize(receiverQueueSize) + .subscriptionName(subName) + .enableBatchIndexAcknowledgment(true) + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(true); + + // Send 100 messages. + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + CompletableFuture lastSent = null; + for (int i = 0; i < 100; i++) { + lastSent = producer.sendAsync(i + ""); + } + producer.flush(); + lastSent.join(); + + // When consumer1 is closed, may some messages are in the client memory(it they are being acked now). + Consumer consumer1 = consumerBuilder.consumerName("c1").subscribe(); + Message[] messagesInClientMemory = new Message[2]; + for (int i = 0; i < 2; i++) { + Message msg = consumer1.receive(2, TimeUnit.SECONDS); + assertNotNull(msg); + messagesInClientMemory[i] = msg; + } + ConsumerImpl consumer2 = (ConsumerImpl) consumerBuilder.consumerName("c2").subscribe(); + Awaitility.await().until(() -> consumer2.isConnected()); + + // The consumer2 will receive messages after consumer1 closed. + // Insert a delay mechanism to make the flow like below: + // 1. Close consumer1, then the 100 messages will be redelivered. + // 2. Read redeliver messages. No messages were acked at this time. + // 3. The in-flight ack of two messages is finished. + // 4. Send the messages to consumer2, consumer2 will get all the 100 messages. + CompletableFuture receiveMessageSignal2 = new CompletableFuture<>(); + org.apache.pulsar.broker.service.Consumer serviceConsumer2 = + makeConsumerReceiveMessagesDelay(topicName, subName, "c2", receiveMessageSignal2); + // step 1: close consumer. + consumer1.close(); + // step 2: wait for read messages from replay queue. + Thread.sleep(2 * 1000); + // step 3: wait for the in-flight ack. + BitSetRecyclable bitSetRecyclable = createBitSetRecyclable(100); + long ledgerId = 0, entryId = 0; + for (Message message : messagesInClientMemory) { + BatchMessageIdImpl msgId = (BatchMessageIdImpl) message.getMessageId(); + bitSetRecyclable.clear(msgId.getBatchIndex()); + ledgerId = msgId.getLedgerId(); + entryId = msgId.getEntryId(); + } + getCursor(topicName, subName).delete(PositionImpl.get(ledgerId, entryId, bitSetRecyclable.toLongArray())); + // step 4: send messages to consumer2. + receiveMessageSignal2.complete(null); + // Verify: Consumer2 will get all the 100 messages, and "unAckMessages" is 100. + List messages2 = new ArrayList<>(); + while (true) { + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + messages2.add(msg); + } + assertEquals(messages2.size(), 100); + assertEquals(serviceConsumer2.getUnackedMessages(), 100); + // After the messages were pop out, the permits in the client memory went to 100. + Awaitility.await().untilAsserted(() -> { + assertEquals(serviceConsumer2.getAvailablePermits() + consumer2.getAvailablePermits(), + receiverQueueSize); + }); + + // cleanup. + producer.close(); + consumer2.close(); + admin.topics().delete(topicName, false); + } + + private BitSetRecyclable createBitSetRecyclable(int batchSize) { + BitSetRecyclable bitSetRecyclable = new BitSetRecyclable(batchSize); + for (int i = 0; i < batchSize; i++) { + bitSetRecyclable.set(i); + } + return bitSetRecyclable; + } + + private ManagedCursorImpl getCursor(String topic, String sub) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + return (ManagedCursorImpl) dispatcher.getCursor(); + } + + /*** + * After {@param signal} complete, the consumer({@param consumerName}) start to receive messages. + */ + private org.apache.pulsar.broker.service.Consumer makeConsumerReceiveMessagesDelay(String topic, String sub, + String consumerName, + CompletableFuture signal) throws Exception { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService().getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + org.apache.pulsar.broker.service.Consumer serviceConsumer = null; + for (org.apache.pulsar.broker.service.Consumer c : dispatcher.getConsumers()){ + if (c.consumerName().equals(consumerName)) { + serviceConsumer = c; + break; + } + } + final org.apache.pulsar.broker.service.Consumer originalConsumer = serviceConsumer; + + // Insert a delay signal. + org.apache.pulsar.broker.service.Consumer spyServiceConsumer = spy(originalConsumer); + doAnswer(invocation -> { + List entries = (List) invocation.getArguments()[0]; + EntryBatchSizes batchSizes = (EntryBatchSizes) invocation.getArguments()[1]; + EntryBatchIndexesAcks batchIndexesAcks = (EntryBatchIndexesAcks) invocation.getArguments()[2]; + int totalMessages = (int) invocation.getArguments()[3]; + long totalBytes = (long) invocation.getArguments()[4]; + long totalChunkedMessages = (long) invocation.getArguments()[5]; + RedeliveryTracker redeliveryTracker = (RedeliveryTracker) invocation.getArguments()[6]; + return signal.thenApply(__ -> originalConsumer.sendMessages(entries, batchSizes, batchIndexesAcks, totalMessages, totalBytes, + totalChunkedMessages, redeliveryTracker)).join(); + }).when(spyServiceConsumer) + .sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any()); + doAnswer(invocation -> { + List entries = (List) invocation.getArguments()[0]; + EntryBatchSizes batchSizes = (EntryBatchSizes) invocation.getArguments()[1]; + EntryBatchIndexesAcks batchIndexesAcks = (EntryBatchIndexesAcks) invocation.getArguments()[2]; + int totalMessages = (int) invocation.getArguments()[3]; + long totalBytes = (long) invocation.getArguments()[4]; + long totalChunkedMessages = (long) invocation.getArguments()[5]; + RedeliveryTracker redeliveryTracker = (RedeliveryTracker) invocation.getArguments()[6]; + long epoch = (long) invocation.getArguments()[7]; + return signal.thenApply(__ -> originalConsumer.sendMessages(entries, batchSizes, batchIndexesAcks, totalMessages, totalBytes, + totalChunkedMessages, redeliveryTracker, epoch)).join(); + }).when(spyServiceConsumer) + .sendMessages(anyList(), any(), any(), anyInt(), anyLong(), anyLong(), any(), anyLong()); + + // Replace the consumer. + Field fConsumerList = AbstractDispatcherMultipleConsumers.class.getDeclaredField("consumerList"); + Field fConsumerSet = AbstractDispatcherMultipleConsumers.class.getDeclaredField("consumerSet"); + fConsumerList.setAccessible(true); + fConsumerSet.setAccessible(true); + List consumerList = + (List) fConsumerList.get(dispatcher); + ObjectSet consumerSet = + (ObjectSet) fConsumerSet.get(dispatcher); + + consumerList.remove(originalConsumer); + consumerSet.removeAll(originalConsumer); + consumerList.add(spyServiceConsumer); + consumerSet.add(spyServiceConsumer); + return originalConsumer; + } } From 071e26da7b3a5b9e07d118098959bb38a81cf0af Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 26 Feb 2024 22:45:55 +0800 Subject: [PATCH 478/494] [fix] [client] fix huge permits if acked a half batched message (#22091) (cherry picked from commit 0c49cac105ee391f327b5d85a02e69ab0a6310a6) --- .../BatchMessageWithBatchIndexLevelTest.java | 85 +++++++++++++++++++ .../pulsar/client/impl/ConsumerBase.java | 5 ++ .../pulsar/client/impl/ConsumerImpl.java | 11 ++- 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java index 3a4cee7f2be83..8e902d5d1e700 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BatchMessageWithBatchIndexLevelTest.java @@ -583,4 +583,89 @@ private org.apache.pulsar.broker.service.Consumer makeConsumerReceiveMessagesDel consumerSet.add(spyServiceConsumer); return originalConsumer; } + + /*** + * 1. Send a batch message contains 100 single messages. + * 2. Ack 2 messages. + * 3. Redeliver the batch message and ack them. + * 4. Verify: the permits is correct. + */ + @Test + public void testPermitsIfHalfAckBatchMessage() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/ns-abc/tp"); + final String subName = "s1"; + final int receiverQueueSize = 1000; + final int ackedMessagesCountInTheFistStep = 2; + admin.topics().createNonPartitionedTopic(topicName); + admin.topics(). createSubscription(topicName, subName, MessageId.earliest); + ConsumerBuilder consumerBuilder = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .receiverQueueSize(receiverQueueSize) + .subscriptionName(subName) + .enableBatchIndexAcknowledgment(true) + .subscriptionType(SubscriptionType.Shared) + .isAckReceiptEnabled(true); + + // Send 100 messages. + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableBatching(true) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .create(); + CompletableFuture lastSent = null; + for (int i = 1; i <= 100; i++) { + lastSent = producer. sendAsync(i + ""); + } + producer.flush(); + lastSent.join(); + + // Ack 2 messages, and trigger a redelivery. + Consumer consumer1 = consumerBuilder.subscribe(); + for (int i = 0; i < ackedMessagesCountInTheFistStep; i++) { + Message msg = consumer1. receive(2, TimeUnit.SECONDS); + assertNotNull(msg); + consumer1.acknowledge(msg); + } + consumer1.close(); + + // Receive the left 98 messages, and ack them. + // Verify the permits is correct. + ConsumerImpl consumer2 = (ConsumerImpl) consumerBuilder.subscribe(); + Awaitility.await().until(() -> consumer2.isConnected()); + List messages = new ArrayList<>(); + int nextMessageValue = ackedMessagesCountInTheFistStep + 1; + while (true) { + Message msg = consumer2.receive(2, TimeUnit.SECONDS); + if (msg == null) { + break; + } + assertEquals(msg.getValue(), nextMessageValue + ""); + messages.add(msg.getMessageId()); + nextMessageValue++; + } + assertEquals(messages.size(), 98); + consumer2.acknowledge(messages); + + org.apache.pulsar.broker.service.Consumer serviceConsumer2 = + getTheUniqueServiceConsumer(topicName, subName); + Awaitility.await().untilAsserted(() -> { + // After the messages were pop out, the permits in the client memory went to 98. + int permitsInClientMemory = consumer2.getAvailablePermits(); + int permitsInBroker = serviceConsumer2.getAvailablePermits(); + assertEquals(permitsInClientMemory + permitsInBroker, receiverQueueSize); + }); + + // cleanup. + producer.close(); + consumer2.close(); + admin.topics().delete(topicName, false); + } + + private org.apache.pulsar.broker.service.Consumer getTheUniqueServiceConsumer(String topic, String sub) { + PersistentTopic persistentTopic = + (PersistentTopic) pulsar.getBrokerService(). getTopic(topic, false).join().get(); + PersistentDispatcherMultipleConsumers dispatcher = + (PersistentDispatcherMultipleConsumers) persistentTopic.getSubscription(sub).getDispatcher(); + return dispatcher.getConsumers().iterator().next(); + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index fec428824c205..67bddf525c6b4 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -64,6 +64,7 @@ import org.apache.pulsar.common.api.proto.CommandSubscribe; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.collections.BitSetRecyclable; import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.slf4j.Logger; @@ -1266,6 +1267,10 @@ protected boolean isValidConsumerEpoch(MessageImpl message) { return true; } + protected boolean isSingleMessageAcked(BitSetRecyclable ackBitSet, int batchIndex) { + return ackBitSet != null && !ackBitSet.get(batchIndex); + } + public boolean hasBatchReceiveTimeout() { return batchReceiveTimeout != null; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index dba54c8d3a3c3..bfe8bd54849ea 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -1181,7 +1181,7 @@ protected MessageImpl newSingleMessage(final int index, return null; } - if (ackBitSet != null && !ackBitSet.get(index)) { + if (isSingleMessageAcked(ackBitSet, index)) { return null; } @@ -1631,7 +1631,14 @@ void receiveIndividualMessagesFromBatch(BrokerEntryMetadata brokerEntryMetadata, singleMessageMetadata, uncompressedPayload, batchMessage, schema, true, ackBitSet, ackSetInMessageId, redeliveryCount, consumerEpoch); if (message == null) { - skippedMessages++; + // If it is not in ackBitSet, it means Broker does not want to deliver it to the client, and + // did not decrease the permits in the broker-side. + // So do not acquire more permits for this message. + // Why not skip this single message in the first line of for-loop block? We need call + // "newSingleMessage" to move "payload.readerIndex" to a correct value to get the correct data. + if (!isSingleMessageAcked(ackBitSet, i)) { + skippedMessages++; + } continue; } if (possibleToDeadLetter != null) { From 90416ab6d07c1c79300fd580d4fe3150e8419d44 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 26 Feb 2024 23:40:02 +0800 Subject: [PATCH 479/494] [fix][sec] Add a check for the input time value (#22023) (cherry picked from commit 31ed115d0b58b599ab87ecff23b9c6b0f5ba2b44) --- .../org/apache/bookkeeper/mledger/ManagedLedgerConfig.java | 5 ++++- .../mledger/impl/ManagedLedgerFactoryMBeanImpl.java | 2 ++ .../bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java | 2 ++ .../java/org/apache/pulsar/broker/stats/DimensionStats.java | 2 ++ .../broker/stats/prometheus/metrics/LongAdderCounter.java | 2 ++ .../apache/pulsar/compaction/CompactionRetentionTest.java | 4 +++- .../org/apache/pulsar/client/api/ClientConfiguration.java | 1 + .../org/apache/pulsar/client/api/ConsumerConfiguration.java | 1 + .../pulsar/client/admin/internal/PulsarAdminBuilderImpl.java | 4 ++++ .../pulsar/client/admin/internal/TransactionsImpl.java | 1 + .../org/apache/pulsar/client/impl/AutoClusterFailover.java | 1 + .../org/apache/pulsar/client/impl/ClientBuilderImpl.java | 2 ++ .../java/org/apache/pulsar/client/impl/ConsumerBase.java | 4 ++++ .../org/apache/pulsar/client/impl/ConsumerBuilderImpl.java | 1 + .../apache/pulsar/client/impl/ControlledClusterFailover.java | 1 + .../org/apache/pulsar/client/impl/ReaderBuilderImpl.java | 1 + .../pulsar/client/impl/TransactionMetaStoreHandler.java | 2 ++ .../apache/pulsar/client/impl/TypedMessageBuilderImpl.java | 1 + .../client/impl/transaction/TransactionBuilderImpl.java | 2 ++ .../main/java/org/apache/pulsar/client/util/ObjectCache.java | 2 ++ .../org/apache/bookkeeper/client/PulsarMockBookKeeper.java | 2 ++ .../java/org/apache/bookkeeper/client/TestStatsProvider.java | 2 ++ 22 files changed, 43 insertions(+), 2 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java index 0c93a5b642cf6..6ee9c2f949243 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java @@ -170,6 +170,7 @@ public int getMinimumRolloverTimeMs() { * the time unit */ public void setMinimumRolloverTime(int minimumRolloverTime, TimeUnit unit) { + checkArgument(minimumRolloverTime >= 0); this.minimumRolloverTimeMs = (int) unit.toMillis(minimumRolloverTime); checkArgument(maximumRolloverTimeMs >= minimumRolloverTimeMs, "Minimum rollover time needs to be less than maximum rollover time"); @@ -195,6 +196,7 @@ public long getMaximumRolloverTimeMs() { * the time unit */ public void setMaximumRolloverTime(int maximumRolloverTime, TimeUnit unit) { + checkArgument(maximumRolloverTime >= 0); this.maximumRolloverTimeMs = unit.toMillis(maximumRolloverTime); checkArgument(maximumRolloverTimeMs >= minimumRolloverTimeMs, "Maximum rollover time needs to be greater than minimum rollover time"); @@ -411,7 +413,8 @@ public ManagedLedgerConfig setThrottleMarkDelete(double throttleMarkDelete) { * time unit for retention time */ public ManagedLedgerConfig setRetentionTime(int retentionTime, TimeUnit unit) { - this.retentionTimeMs = unit.toMillis(retentionTime); + checkArgument(retentionTime >= -1, "The retention time should be -1, 0 or value > 0"); + this.retentionTimeMs = retentionTime != -1 ? unit.toMillis(retentionTime) : -1; return this; } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java index cf3d7142d617e..5a6bc8017b7e0 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; +import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.ManagedLedgerFactoryMXBean; @@ -41,6 +42,7 @@ public ManagedLedgerFactoryMBeanImpl(ManagedLedgerFactoryImpl factory) throws Ex } public void refreshStats(long period, TimeUnit unit) { + checkArgument(period >= 0); double seconds = unit.toMillis(period) / 1000.0; if (seconds <= 0.0) { diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java index e057dee99538e..7884add95526c 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanImpl.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; +import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.mledger.ManagedCursor; @@ -63,6 +64,7 @@ public ManagedLedgerMBeanImpl(ManagedLedgerImpl managedLedger) { } public void refreshStats(long period, TimeUnit unit) { + checkArgument(period >= 0); double seconds = unit.toMillis(period) / 1000.0; if (seconds <= 0.0) { // skip refreshing stats diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java index 604265b554050..815778ecc9575 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/DimensionStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats; +import static com.google.common.base.Preconditions.checkArgument; import static io.prometheus.client.CollectorRegistry.defaultRegistry; import io.prometheus.client.Collector; import io.prometheus.client.Summary; @@ -70,6 +71,7 @@ public DimensionStats(String name, long updateDurationInSec) { } public void recordDimensionTimeValue(long latency, TimeUnit unit) { + checkArgument(latency >= 0); summary.observe(unit.toMillis(latency)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java index 8ade2bc883f9a..c2816f5a2a013 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/metrics/LongAdderCounter.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.stats.prometheus.metrics; +import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.apache.bookkeeper.stats.Counter; @@ -57,6 +58,7 @@ public void addCount(long delta) { @Override public void addLatency(long eventLatency, TimeUnit unit) { + checkArgument(eventLatency >= 0); long valueMillis = unit.toMillis(eventLatency); counter.add(valueMillis); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java index 98bf2b819c2ba..88d923f74e196 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionRetentionTest.java @@ -257,7 +257,9 @@ private void checkTopicRetentionPolicy(String topicName, RetentionPolicies reten ManagedLedgerConfig config = pulsar.getBrokerService() .getManagedLedgerConfig(TopicName.get(topicName)).get(); Assert.assertEquals(config.getRetentionSizeInMB(), retentionPolicies.getRetentionSizeInMB()); - Assert.assertEquals(config.getRetentionTimeMillis(),retentionPolicies.getRetentionTimeInMinutes() * 60000L); + Assert.assertEquals(config.getRetentionTimeMillis(), retentionPolicies.getRetentionTimeInMinutes() < 0 + ? retentionPolicies.getRetentionTimeInMinutes() + : retentionPolicies.getRetentionTimeInMinutes() * 60000L); } private void testCompactionCursorRetention(String topic) throws Exception { diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java index 3b0efe64cf588..ea2bba166e6c5 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ClientConfiguration.java @@ -368,6 +368,7 @@ public ClientConfiguration setServiceUrl(String serviceUrl) { * @param unit the time unit in which the duration is defined */ public void setConnectionTimeout(int duration, TimeUnit unit) { + checkArgument(duration >= 0); confData.setConnectionTimeoutMs((int) unit.toMillis(duration)); } diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ConsumerConfiguration.java b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ConsumerConfiguration.java index 81956db56f774..f2101b287043c 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ConsumerConfiguration.java +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/java/org/apache/pulsar/client/api/ConsumerConfiguration.java @@ -69,6 +69,7 @@ public long getAckTimeoutMillis() { * @return {@link ConsumerConfiguration} */ public ConsumerConfiguration setAckTimeout(long ackTimeout, TimeUnit timeUnit) { + checkArgument(ackTimeout >= 0); long ackTimeoutMillis = timeUnit.toMillis(ackTimeout); checkArgument(ackTimeoutMillis >= minAckTimeoutMillis, "Ack timeout should be should be greater than " + minAckTimeoutMillis + " ms"); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java index 009fa67fbaa29..a9d913c016490 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.admin.internal; +import static com.google.common.base.Preconditions.checkArgument; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -200,18 +201,21 @@ public PulsarAdminBuilder tlsProtocols(Set tlsProtocols) { @Override public PulsarAdminBuilder connectionTimeout(int connectionTimeout, TimeUnit connectionTimeoutUnit) { + checkArgument(connectionTimeout >= 0); this.conf.setConnectionTimeoutMs((int) connectionTimeoutUnit.toMillis(connectionTimeout)); return this; } @Override public PulsarAdminBuilder readTimeout(int readTimeout, TimeUnit readTimeoutUnit) { + checkArgument(readTimeout >= 0); this.conf.setReadTimeoutMs((int) readTimeoutUnit.toMillis(readTimeout)); return this; } @Override public PulsarAdminBuilder requestTimeout(int requestTimeout, TimeUnit requestTimeoutUnit) { + checkArgument(requestTimeout >= 0); this.conf.setRequestTimeoutMs((int) requestTimeoutUnit.toMillis(requestTimeout)); return this; } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java index 5693ebc8f60aa..835fecfc8e195 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/TransactionsImpl.java @@ -164,6 +164,7 @@ public TransactionPendingAckStats getPendingAckStats(String topic, String subNam @Override public CompletableFuture> getSlowTransactionsByCoordinatorIdAsync( Integer coordinatorId, long timeout, TimeUnit timeUnit) { + checkArgument(timeout >= 0); WebTarget path = adminV3Transactions.path("slowTransactions"); path = path.path(timeUnit.toMillis(timeout) + ""); if (coordinatorId != null) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java index 68b781e67d29c..a1017e66760a5 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AutoClusterFailover.java @@ -329,6 +329,7 @@ public AutoClusterFailoverBuilder switchBackDelay(long switchBackDelay, TimeUnit @Override public AutoClusterFailoverBuilder checkInterval(long interval, TimeUnit timeUnit) { + checkArgument(interval >= 0L, "check interval time must not be negative."); this.checkIntervalMs = timeUnit.toMillis(interval); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java index 7677045f0899b..107f15905044c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java @@ -166,6 +166,7 @@ public ClientBuilder operationTimeout(int operationTimeout, TimeUnit unit) { @Override public ClientBuilder lookupTimeout(int lookupTimeout, TimeUnit unit) { + checkArgument(lookupTimeout >= 0, "lookupTimeout must not be negative"); conf.setLookupTimeoutMs(unit.toMillis(lookupTimeout)); return this; } @@ -331,6 +332,7 @@ public ClientBuilder keepAliveInterval(int keepAliveInterval, TimeUnit unit) { @Override public ClientBuilder connectionTimeout(int duration, TimeUnit unit) { + checkArgument(duration >= 0, "connectionTimeout needs to be >= 0"); conf.setConnectionTimeoutMs((int) unit.toMillis(duration)); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java index 67bddf525c6b4..79cd8db4b99bb 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBase.java @@ -449,6 +449,7 @@ public void acknowledge(Messages messages) throws PulsarClientException { @Override public void reconsumeLater(Message message, long delayTime, TimeUnit unit) throws PulsarClientException { + checkArgument(delayTime >= 0, "The delay time must not be negative."); reconsumeLater(message, null, delayTime, unit); } @@ -563,6 +564,7 @@ public CompletableFuture reconsumeLaterAsync(Message message, long dela @Override public CompletableFuture reconsumeLaterAsync( Message message, Map customProperties, long delayTime, TimeUnit unit) { + checkArgument(delayTime >= 0, "The delay time must not be negative."); if (!conf.isRetryEnable()) { return FutureUtil.failedFuture(new PulsarClientException(RECONSUME_LATER_ERROR_MSG)); } @@ -599,12 +601,14 @@ public CompletableFuture acknowledgeCumulativeAsync(Message message) { @Override public CompletableFuture reconsumeLaterCumulativeAsync(Message message, long delayTime, TimeUnit unit) { + checkArgument(delayTime >= 0, "The delay time must not be negative."); return reconsumeLaterCumulativeAsync(message, null, delayTime, unit); } @Override public CompletableFuture reconsumeLaterCumulativeAsync( Message message, Map customProperties, long delayTime, TimeUnit unit) { + checkArgument(delayTime >= 0, "The delay time must not be negative."); if (!conf.isRetryEnable()) { return FutureUtil.failedFuture(new PulsarClientException(RECONSUME_LATER_ERROR_MSG)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java index f644c6a18398f..895273a90c050 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerBuilderImpl.java @@ -497,6 +497,7 @@ public ConsumerBuilder enableBatchIndexAcknowledgment(boolean batchIndexAckno @Override public ConsumerBuilder expireTimeOfIncompleteChunkedMessage(long duration, TimeUnit unit) { + checkArgument(duration >= 0, "expired time of incomplete chunk message must not be negative"); conf.setExpireTimeOfIncompleteChunkedMessageMillis(unit.toMillis(duration)); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java index 080d328e3f02c..9d30108ec7a1d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ControlledClusterFailover.java @@ -236,6 +236,7 @@ public ControlledClusterFailoverBuilder urlProviderHeader(Map he @Override public ControlledClusterFailoverBuilder checkInterval(long interval, @NonNull TimeUnit timeUnit) { + checkArgument(interval >= 0, "The check interval time must not be negative."); this.interval = timeUnit.toMillis(interval); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java index ca2011cf18a42..bed0f49f16068 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ReaderBuilderImpl.java @@ -267,6 +267,7 @@ public ReaderBuilder autoAckOldestChunkedMessageOnQueueFull(boolean autoAckOl @Override public ReaderBuilder expireTimeOfIncompleteChunkedMessage(long duration, TimeUnit unit) { + checkArgument(duration >= 0, "The expired time must not be negative."); conf.setExpireTimeOfIncompleteChunkedMessageMillis(unit.toMillis(duration)); return this; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index ebbfca0c3cb3f..bf5c1f556de8c 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl; +import static com.google.common.base.Preconditions.checkArgument; import com.google.common.annotations.VisibleForTesting; import io.netty.buffer.ByteBuf; import io.netty.util.Recycler; @@ -203,6 +204,7 @@ private void failPendingRequest() { } public CompletableFuture newTransactionAsync(long timeout, TimeUnit unit) { + checkArgument(timeout >= 0, "The timeout must not be negative."); if (LOG.isDebugEnabled()) { LOG.debug("New transaction with timeout in ms {}", unit.toMillis(timeout)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java index 026f8a1e69e0b..895949fdf32cc 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TypedMessageBuilderImpl.java @@ -212,6 +212,7 @@ public TypedMessageBuilder disableReplication() { @Override public TypedMessageBuilder deliverAfter(long delay, TimeUnit unit) { + checkArgument(delay >= 0, "The delay time must not be negative."); return deliverAt(System.currentTimeMillis() + unit.toMillis(delay)); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java index c5e9d4781c56f..255dedcbb85e9 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/transaction/TransactionBuilderImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.impl.transaction; +import static com.google.common.base.Preconditions.checkArgument; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; @@ -45,6 +46,7 @@ public TransactionBuilderImpl(PulsarClientImpl client, TransactionCoordinatorCli @Override public TransactionBuilder withTransactionTimeout(long txnTimeout, TimeUnit timeoutUnit) { + checkArgument(txnTimeout >= 0, "The txn timeout must not be negative."); this.txnTimeout = txnTimeout; this.timeUnit = timeoutUnit; return this; diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ObjectCache.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ObjectCache.java index dc057ffe32daf..cf0620edf98df 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/ObjectCache.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/ObjectCache.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.client.util; +import static com.google.common.base.Preconditions.checkArgument; import java.time.Clock; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -33,6 +34,7 @@ public class ObjectCache implements Supplier { public ObjectCache(Supplier supplier, long cacheDuration, TimeUnit unit) { this(supplier, cacheDuration, unit, Clock.systemUTC()); + checkArgument(cacheDuration >= 0, "The cache duration must not be negative."); } ObjectCache(Supplier supplier, long cacheDuration, TimeUnit unit, Clock clock) { diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java index f0d279ef25050..998ef73fbd3e9 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.client; +import static com.google.common.base.Preconditions.checkArgument; import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.Arrays; @@ -364,6 +365,7 @@ public synchronized CompletableFuture promiseAfter(int steps, List= 0, "The delay time must not be negative."); addEntryDelaysMillis.add(unit.toMillis(delay)); } diff --git a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java index 4d08a7f80df5b..cf08cc635106e 100644 --- a/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java +++ b/testmocks/src/main/java/org/apache/bookkeeper/client/TestStatsProvider.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.client; +import static com.google.common.base.Preconditions.checkArgument; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -65,6 +66,7 @@ public void addCount(long delta) { @Override public void addLatency(long eventLatency, TimeUnit unit) { + checkArgument(eventLatency >= 0, "The event latency must not be negative."); long valueMillis = unit.toMillis(eventLatency); updateMax(val.addAndGet(valueMillis)); } From 3b43433554ebdf1b0f6810e4b93d8a90433c7e38 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 27 Feb 2024 22:22:10 +0800 Subject: [PATCH 480/494] [fix][test] Fix test testAsyncFunctionMaxPending (#22121) (cherry picked from commit 91de98ad45675c23b79c18e49602e9c49ec880b3) --- .../pulsar/functions/instance/JavaInstanceTest.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java index efe80922dfa8c..5a3332042938d 100644 --- a/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java +++ b/pulsar-functions/instance/src/test/java/org/apache/pulsar/functions/instance/JavaInstanceTest.java @@ -24,6 +24,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertSame; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import lombok.Cleanup; @@ -185,6 +186,7 @@ public void testUserExceptionThrowingAsyncFunction() throws Exception { @Test public void testAsyncFunctionMaxPending() throws Exception { + CountDownLatch count = new CountDownLatch(1); InstanceConfig instanceConfig = new InstanceConfig(); int pendingQueueSize = 3; instanceConfig.setMaxPendingAsyncRequests(pendingQueueSize); @@ -196,7 +198,7 @@ public void testAsyncFunctionMaxPending() throws Exception { CompletableFuture result = new CompletableFuture<>(); executor.submit(() -> { try { - Thread.sleep(500); + count.await(); result.complete(String.format("%s-lambda", input)); } catch (Exception e) { result.completeExceptionally(e); @@ -222,8 +224,13 @@ public void testAsyncFunctionMaxPending() throws Exception { // no space left assertEquals(0, instance.getPendingAsyncRequests().remainingCapacity()); + AsyncFuncRequest[] asyncFuncRequests = new AsyncFuncRequest[3]; for (int i = 0; i < 3; i++) { - AsyncFuncRequest request = instance.getPendingAsyncRequests().poll(); + asyncFuncRequests[i] = instance.getPendingAsyncRequests().poll(); + } + + count.countDown(); + for (AsyncFuncRequest request : asyncFuncRequests) { Assert.assertEquals(request.getProcessResult().get(), testString + "-lambda"); } From f26b1bfafcfb062144a245fc8226a79543c88a8c Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:38:40 +0800 Subject: [PATCH 481/494] [fix][test] fix test testSyncNormalPositionWhenTBRecover (#22120) 1. Change to None state before invoking the recovery. 2. Improve the method `checkTopicTransactionBufferState` to see the test result easier. ``` org.awaitility.core.ConditionTimeoutException: Condition with org.apache.pulsar.broker.transaction.buffer.TransactionStablePositionTest was not fulfilled within 10 seconds. at org.awaitility.core.ConditionAwaiter.await(ConditionAwaiter.java:167) at org.awaitility.core.CallableCondition.await(CallableCondition.java:78) at org.awaitility.core.CallableCondition.await(CallableCondition.java:26) at org.awaitility.core.ConditionFactory.until(ConditionFactory.java:985) at org.awaitility.core.ConditionFactory.until(ConditionFactory.java:954) at org.apache.pulsar.broker.transaction.buffer.TransactionStablePositionTest.checkTopicTransactionBufferState(TransactionStablePositionTest.java:239) at org.apache.pulsar.broker.transaction.buffer.TransactionStablePositionTest.testSyncNormalPositionWhenTBRecover(TransactionStablePositionTest.java:229) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.testng.internal.invokers.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:139) at org.testng.internal.invokers.TestInvoker.invokeMethod(TestInvoker.java:677) at org.testng.internal.invokers.TestInvoker.invokeTestMethod(TestInvoker.java:221) at org.testng.internal.invokers.MethodRunner.runInSequence(MethodRunner.java:50) at org.testng.internal.invokers.TestInvoker$MethodInvocationAgent.invoke(TestInvoker.java:969) at org.testng.internal.invokers.TestInvoker.invokeTestMethods(TestInvoker.java:194) at org.testng.internal.invokers.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:148) at org.testng.internal.invokers.TestMethodWorker.run(TestMethodWorker.java:128) at java.base/java.util.ArrayList.forEach(ArrayList.java:1511) at org.testng.TestRunner.privateRun(TestRunner.java:829) at org.testng.TestRunner.run(TestRunner.java:602) at org.testng.SuiteRunner.runTest(SuiteRunner.java:437) at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:431) at org.testng.SuiteRunner.privateRun(SuiteRunner.java:391) at org.testng.SuiteRunner.run(SuiteRunner.java:330) at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52) at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:95) at org.testng.TestNG.runSuitesSequentially(TestNG.java:1256) at org.testng.TestNG.runSuitesLocally(TestNG.java:1176) at org.testng.TestNG.runSuites(TestNG.java:1099) at org.testng.TestNG.run(TestNG.java:1067) at com.intellij.rt.testng.IDEARemoteTestNG.run(IDEARemoteTestNG.java:65) at com.intellij.rt.testng.RemoteTestNGStarter.main(RemoteTestNGStarter.java:105) ``` 1. Change to None state before invoking the recovery. 2. Improve the method `checkTopicTransactionBufferState` to see the test result easier. (cherry picked from commit 30134966a1812506517971b2ba8148b4b511dddb) --- .../buffer/TransactionStablePositionTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java index 55d115905a35d..2297ae79fafa7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/buffer/TransactionStablePositionTest.java @@ -213,14 +213,14 @@ public void testSyncNormalPositionWhenTBRecover(boolean clientEnableTransaction, position = topicTransactionBuffer.getMaxReadPosition(); assertEquals(position, PositionImpl.EARLIEST); + // change to None state can recover + field.set(topicTransactionBuffer, TopicTransactionBufferState.State.None); + // invoke recover Method method = TopicTransactionBuffer.class.getDeclaredMethod("recover"); method.setAccessible(true); method.invoke(topicTransactionBuffer); - // change to None state can recover - field.set(topicTransactionBuffer, TopicTransactionBufferState.State.None); - // recover success again checkTopicTransactionBufferState(clientEnableTransaction, topicTransactionBuffer); @@ -232,13 +232,15 @@ public void testSyncNormalPositionWhenTBRecover(boolean clientEnableTransaction, private void checkTopicTransactionBufferState(boolean clientEnableTransaction, TopicTransactionBuffer topicTransactionBuffer) { // recover success - Awaitility.await().until(() -> { + Awaitility.await().untilAsserted(() -> { if (clientEnableTransaction) { // recover success, client enable transaction will change to Ready State - return topicTransactionBuffer.getStats(false).state.equals(Ready.name()); + assertEquals(topicTransactionBuffer.getStats(false).state, + Ready.name()); } else { // recover success, client disable transaction will change to NoSnapshot State - return topicTransactionBuffer.getStats(false).state.equals(NoSnapshot.name()); + assertEquals(topicTransactionBuffer.getStats(false).state, + NoSnapshot.name()); } }); } From 1fd7934aca25a0b02235e46e3f087248123109e5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 11 Oct 2023 10:13:26 +0300 Subject: [PATCH 482/494] [fix][ml] Make mlOwnershipChecker asynchronous so that it doesn't block/deadlock threads (#21333) (cherry picked from commit eb9fa63d6bcaa4e1cbc4b87e36ead5a3ff6c44ae) --- .../mledger/ManagedLedgerFactory.java | 2 +- .../mledger/impl/ManagedCursorImpl.java | 57 ++++++++++++------- .../impl/ManagedLedgerFactoryImpl.java | 2 +- .../mledger/impl/ManagedLedgerImpl.java | 4 +- .../mledger/impl/ShadowManagedLedgerImpl.java | 3 +- .../mledger/impl/ManagedLedgerTest.java | 2 +- .../pulsar/broker/service/BrokerService.java | 19 ++++--- .../impl/MLPendingAckStoreProvider.java | 2 +- .../broker/admin/TopicPoliciesTest.java | 2 +- .../OwnerShipCacheForCurrentServerTest.java | 2 +- 10 files changed, 57 insertions(+), 38 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java index b1427bab80b22..e09fd84ea55f2 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java @@ -90,7 +90,7 @@ ManagedLedger open(String name, ManagedLedgerConfig config) * opaque context */ void asyncOpen(String name, ManagedLedgerConfig config, OpenLedgerCallback callback, - Supplier mlOwnershipChecker, Object ctx); + Supplier> mlOwnershipChecker, Object ctx); /** * Open a {@link ReadOnlyCursor} positioned to the earliest entry for the specified managed ledger. diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java index d0fdfe50165d3..10e72f709feed 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java @@ -2691,32 +2691,47 @@ public void operationComplete(Void result, Stat stat) { } @Override - public void operationFailed(MetaStoreException e) { - if (e instanceof MetaStoreException.BadVersionException) { + public void operationFailed(MetaStoreException topLevelException) { + if (topLevelException instanceof MetaStoreException.BadVersionException) { log.warn("[{}] Failed to update cursor metadata for {} due to version conflict {}", - ledger.name, name, e.getMessage()); + ledger.name, name, topLevelException.getMessage()); // it means previous owner of the ml might have updated the version incorrectly. So, check // the ownership and refresh the version again. - if (ledger.mlOwnershipChecker != null && ledger.mlOwnershipChecker.get()) { - ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, - new MetaStoreCallback() { - @Override - public void operationComplete(ManagedCursorInfo info, Stat stat) { - updateCursorLedgerStat(info, stat); - } - - @Override - public void operationFailed(MetaStoreException e) { - if (log.isDebugEnabled()) { - log.debug( - "[{}] Failed to refresh cursor metadata-version for {} due " - + "to {}", ledger.name, name, e.getMessage()); - } - } - }); + if (ledger.mlOwnershipChecker != null) { + ledger.mlOwnershipChecker.get().whenComplete((hasOwnership, t) -> { + if (t == null && hasOwnership) { + ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, + new MetaStoreCallback<>() { + @Override + public void operationComplete(ManagedCursorInfo info, Stat stat) { + updateCursorLedgerStat(info, stat); + // fail the top level call so that the caller can retry + callback.operationFailed(topLevelException); + } + + @Override + public void operationFailed(MetaStoreException e) { + if (log.isDebugEnabled()) { + log.debug( + "[{}] Failed to refresh cursor metadata-version " + + "for {} due to {}", ledger.name, name, + e.getMessage()); + } + // fail the top level call so that the caller can retry + callback.operationFailed(topLevelException); + } + }); + } else { + // fail the top level call so that the caller can retry + callback.operationFailed(topLevelException); + } + }); + } else { + callback.operationFailed(topLevelException); } + } else { + callback.operationFailed(topLevelException); } - callback.operationFailed(e); } }); } diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java index 85144ba12316c..ce76717942d41 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java @@ -329,7 +329,7 @@ public void asyncOpen(String name, OpenLedgerCallback callback, Object ctx) { @Override public void asyncOpen(final String name, final ManagedLedgerConfig config, final OpenLedgerCallback callback, - Supplier mlOwnershipChecker, final Object ctx) { + Supplier> mlOwnershipChecker, final Object ctx) { if (closed) { callback.openLedgerFailed(new ManagedLedgerException.ManagedLedgerFactoryClosedException(), ctx); return; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 37efc72c32c23..6727fc63479b4 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -232,7 +232,7 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback { private static final Random random = new Random(System.currentTimeMillis()); private long maximumRolloverTimeMs; - protected final Supplier mlOwnershipChecker; + protected final Supplier> mlOwnershipChecker; volatile PositionImpl lastConfirmedEntry; @@ -336,7 +336,7 @@ public ManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper } public ManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store, ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, - final String name, final Supplier mlOwnershipChecker) { + final String name, final Supplier> mlOwnershipChecker) { this.factory = factory; this.bookKeeper = bookKeeper; this.config = config; diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java index b33dd87543f77..8b2742d958783 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ShadowManagedLedgerImpl.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.AsyncCallback; @@ -50,7 +51,7 @@ public class ShadowManagedLedgerImpl extends ManagedLedgerImpl { public ShadowManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store, ManagedLedgerConfig config, OrderedScheduler scheduledExecutor, - String name, final Supplier mlOwnershipChecker) { + String name, final Supplier> mlOwnershipChecker) { super(factory, bookKeeper, store, config, scheduledExecutor, name, mlOwnershipChecker); this.sourceMLName = config.getShadowSourceName(); } diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java index 6c4f21c3af29d..34f65dfd00ee8 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java @@ -3399,7 +3399,7 @@ public void openCursorFailed(ManagedLedgerException exception, Object ctx) { @Override public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { } - }, checkOwnershipFlag ? () -> true : null, null); + }, checkOwnershipFlag ? () -> CompletableFuture.completedFuture(true) : null, null); latch.await(); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 8997e1e98ee63..fbec9bf413a50 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1808,7 +1808,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { topicFuture.completeExceptionally(new PersistenceException(exception)); } } - }, () -> isTopicNsOwnedByBroker(topicName), null); + }, () -> isTopicNsOwnedByBrokerAsync(topicName), null); }).exceptionally((exception) -> { log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception); @@ -2200,13 +2200,16 @@ public void monitorBacklogQuota() { }); } - public boolean isTopicNsOwnedByBroker(TopicName topicName) { - try { - return pulsar.getNamespaceService().isServiceUnitOwned(topicName); - } catch (Exception e) { - log.warn("Failed to check the ownership of the topic: {}, {}", topicName, e.getMessage()); - } - return false; + public CompletableFuture isTopicNsOwnedByBrokerAsync(TopicName topicName) { + return pulsar.getNamespaceService().isServiceUnitOwnedAsync(topicName) + .handle((hasOwnership, t) -> { + if (t == null) { + return hasOwnership; + } else { + log.warn("Failed to check the ownership of the topic: {}, {}", topicName, t.getMessage()); + return false; + } + }); } public CompletableFuture checkTopicNsOwnership(final String topic) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java index ecc6599ce52b5..5308648b80c1d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/pendingack/impl/MLPendingAckStoreProvider.java @@ -159,7 +159,7 @@ public void openLedgerFailed(ManagedLedgerException exception, Object ctx) { , originPersistentTopic.getName(), subscription.getName(), exception); pendingAckStoreFuture.completeExceptionally(exception); } - }, () -> true, null); + }, () -> CompletableFuture.completedFuture(true), null); }).exceptionally(e -> { Throwable t = FutureUtil.unwrapCompletionException(e); log.error("[{}] [{}] Failed to get managedLedger config when init pending ack store!", diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index fe0901bf2d53b..c5111b47d2082 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -183,7 +183,7 @@ public void testTopicPolicyInitialValueWithNamespaceAlreadyLoaded() throws Excep //load the nameserver, but topic is not init. log.info("lookup:{}",admin.lookups().lookupTopic(topic)); - assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBroker(topicName)); + assertTrue(pulsar.getBrokerService().isTopicNsOwnedByBrokerAsync(topicName).join()); assertFalse(pulsar.getBrokerService().getTopics().containsKey(topic)); //make sure namespace policy reader is fully started. Awaitility.await().untilAsserted(()-> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipCacheForCurrentServerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipCacheForCurrentServerTest.java index e53d5c25bd2b5..22fa2c32b5655 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipCacheForCurrentServerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipCacheForCurrentServerTest.java @@ -76,7 +76,7 @@ public void testCreateTopicWithNotTopicNsOwnedBroker() { int verifiedBrokerNum = 0; for (PulsarService pulsarService : this.getPulsarServiceList()) { BrokerService bs = pulsarService.getBrokerService(); - if (bs.isTopicNsOwnedByBroker(TopicName.get(topicName))) { + if (bs.isTopicNsOwnedByBrokerAsync(TopicName.get(topicName)).join()) { continue; } verifiedBrokerNum ++; From f308ab2f4a2e359aa2109722d13eefdabef7d1c9 Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Sun, 18 Feb 2024 12:29:51 +0800 Subject: [PATCH 483/494] [improve][broker] Add an error log to troubleshoot the failure of starting broker registry. (#22065) (cherry picked from commit baddda56e27486ea3b803bf7569768164444a738) --- .../loadbalance/extensions/ExtensibleLoadManagerImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 5fd675d7df897..baf3e7b156335 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -390,6 +390,8 @@ public void start() throws PulsarServerException { this.initWaiter.countDown(); this.started = true; } catch (Exception ex) { + log.error("Failed to start the extensible load balance and close broker registry {}.", + this.brokerRegistry, ex); if (this.brokerRegistry != null) { brokerRegistry.close(); } From 6df02655a3c307d29a86b7f3e0ea46c16ad18aea Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 28 Feb 2024 17:52:37 -0800 Subject: [PATCH 484/494] [fix][broker][branch-3.0] Set ServiceUnitStateChannel topic compaction threshold explicitly, improve getOwnerAsync, and fix other bugs (#22064) (#22154) --- .../extensions/ExtensibleLoadManagerImpl.java | 56 ++++++-- .../channel/ServiceUnitStateChannelImpl.java | 101 ++++++++++--- .../store/LoadDataStoreFactory.java | 7 +- .../store/TableViewLoadDataStoreImpl.java | 79 ++++++++-- .../impl/RawBatchMessageContainerImpl.java | 1 + .../ExtensibleLoadManagerImplTest.java | 136 ++++++++++-------- .../channel/ServiceUnitStateChannelTest.java | 77 +++++----- .../extensions/store/LoadDataStoreTest.java | 41 ++---- .../pulsar/client/impl/TableViewImpl.java | 8 +- .../ExtensibleLoadManagerTest.java | 1 - 10 files changed, 325 insertions(+), 182 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index baf3e7b156335..409bb55075be0 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -21,6 +21,7 @@ import static java.lang.String.format; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Follower; import static org.apache.pulsar.broker.loadbalance.extensions.ExtensibleLoadManagerImpl.Role.Leader; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl.TOPIC; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Label.Success; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import static org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.getNamespaceBundle; @@ -117,6 +118,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private static final long MONITOR_INTERVAL_IN_MILLIS = 120_000; + public static final long COMPACTION_THRESHOLD = 5 * 1024 * 1024; + private static final String ELECTION_ROOT = "/loadbalance/extension/leader"; private PulsarService pulsar; @@ -173,6 +176,8 @@ public class ExtensibleLoadManagerImpl implements ExtensibleLoadManager { private volatile boolean started = false; + private boolean configuredSystemTopics = false; + private final AssignCounter assignCounter = new AssignCounter(); @Getter private final UnloadCounter unloadCounter = new UnloadCounter(); @@ -262,6 +267,10 @@ public static boolean isLoadManagerExtensionEnabled(ServiceConfiguration conf) { return ExtensibleLoadManagerImpl.class.getName().equals(conf.getLoadManagerClassName()); } + public static boolean isLoadManagerExtensionEnabled(PulsarService pulsar) { + return pulsar.getLoadManager().get() instanceof ExtensibleLoadManagerWrapper; + } + public static ExtensibleLoadManagerImpl get(LoadManager loadManager) { if (!(loadManager instanceof ExtensibleLoadManagerWrapper loadManagerWrapper)) { throw new IllegalArgumentException("The load manager should be 'ExtensibleLoadManagerWrapper'."); @@ -291,6 +300,27 @@ private static void createSystemTopics(PulsarService pulsar) throws PulsarServer createSystemTopic(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); } + private static boolean configureSystemTopics(PulsarService pulsar) { + try { + if (ExtensibleLoadManagerImpl.isLoadManagerExtensionEnabled(pulsar) + && (pulsar.getConfiguration().isSystemTopicEnabled() + && pulsar.getConfiguration().isTopicLevelPoliciesEnabled())) { + Long threshold = pulsar.getAdminClient().topicPolicies().getCompactionThreshold(TOPIC); + if (threshold == null || COMPACTION_THRESHOLD != threshold.longValue()) { + pulsar.getAdminClient().topicPolicies().setCompactionThreshold(TOPIC, COMPACTION_THRESHOLD); + log.info("Set compaction threshold: {} bytes for system topic {}.", COMPACTION_THRESHOLD, TOPIC); + } + } else { + log.warn("System topic or topic level policies is disabled. " + + "{} compaction threshold follows the broker or namespace policies.", TOPIC); + } + return true; + } catch (Exception e) { + log.error("Failed to set compaction threshold for system topic:{}", TOPIC, e); + } + return false; + } + @Override public void start() throws PulsarServerException { if (this.started) { @@ -329,9 +359,9 @@ public void start() throws PulsarServerException { try { this.brokerLoadDataStore = LoadDataStoreFactory - .create(pulsar.getClient(), BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); + .create(pulsar, BROKER_LOAD_DATA_STORE_TOPIC, BrokerLoadData.class); this.topBundlesLoadDataStore = LoadDataStoreFactory - .create(pulsar.getClient(), TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); + .create(pulsar, TOP_BUNDLES_LOAD_DATA_STORE_TOPIC, TopBundlesLoadData.class); } catch (LoadDataStoreException e) { throw new PulsarServerException(e); } @@ -389,6 +419,7 @@ public void start() throws PulsarServerException { this.splitScheduler.start(); this.initWaiter.countDown(); this.started = true; + log.info("Started load manager."); } catch (Exception ex) { log.error("Failed to start the extensible load balance and close broker registry {}.", this.brokerRegistry, ex); @@ -523,7 +554,7 @@ private CompletableFuture> dedupeLookupRequest( if (ex != null) { assignCounter.incrementFailure(); } - lookupRequests.remove(key, newFutureCreated.getValue()); + lookupRequests.remove(key); }); } } @@ -736,13 +767,13 @@ public void close() throws PulsarServerException { } public static boolean isInternalTopic(String topic) { - return topic.startsWith(ServiceUnitStateChannelImpl.TOPIC) + return topic.startsWith(TOPIC) || topic.startsWith(BROKER_LOAD_DATA_STORE_TOPIC) || topic.startsWith(TOP_BUNDLES_LOAD_DATA_STORE_TOPIC); } @VisibleForTesting - void playLeader() { + synchronized void playLeader() { log.info("This broker:{} is setting the role from {} to {}", pulsar.getBrokerId(), role, Leader); int retry = 0; @@ -760,7 +791,7 @@ void playLeader() { serviceUnitStateChannel.scheduleOwnershipMonitor(); break; } catch (Throwable e) { - log.error("The broker:{} failed to set the role. Retrying {} th ...", + log.warn("The broker:{} failed to set the role. Retrying {} th ...", pulsar.getBrokerId(), ++retry, e); try { Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS)); @@ -780,7 +811,7 @@ void playLeader() { } @VisibleForTesting - void playFollower() { + synchronized void playFollower() { log.info("This broker:{} is setting the role from {} to {}", pulsar.getBrokerId(), role, Follower); int retry = 0; @@ -794,7 +825,7 @@ void playFollower() { topBundlesLoadDataStore.startProducer(); break; } catch (Throwable e) { - log.error("The broker:{} failed to set the role. Retrying {} th ...", + log.warn("The broker:{} failed to set the role. Retrying {} th ...", pulsar.getBrokerId(), ++retry, e); try { Thread.sleep(Math.min(retry * 10, MAX_ROLE_CHANGE_RETRY_DELAY_IN_MILLIS)); @@ -834,7 +865,9 @@ public List getMetrics() { return metricsCollection; } - private void monitor() { + + @VisibleForTesting + protected void monitor() { try { initWaiter.await(); @@ -842,6 +875,11 @@ private void monitor() { // Periodically check the role in case ZK watcher fails. var isChannelOwner = serviceUnitStateChannel.isChannelOwner(); if (isChannelOwner) { + // System topic config might fail due to the race condition + // with topic policy init(Topic policies cache have not init). + if (!configuredSystemTopics) { + configuredSystemTopics = configureSystemTopics(pulsar); + } if (role != Leader) { log.warn("Current role:{} does not match with the channel ownership:{}. " + "Playing the leader role.", role, isChannelOwner); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 8efa4e2e21ad8..220ce02ba5aef 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -492,21 +492,28 @@ private CompletableFuture> getActiveOwnerAsync( String serviceUnit, ServiceUnitState state, Optional owner) { - CompletableFuture> activeOwner = owner.isPresent() - ? brokerRegistry.lookupAsync(owner.get()).thenApply(lookupData -> lookupData.flatMap(__ -> owner)) - : CompletableFuture.completedFuture(Optional.empty()); - - return activeOwner - .thenCompose(broker -> broker - .map(__ -> activeOwner) - .orElseGet(() -> deferGetOwnerRequest(serviceUnit).thenApply(Optional::ofNullable))) - .whenComplete((__, e) -> { + return deferGetOwnerRequest(serviceUnit) + .thenCompose(newOwner -> { + if (newOwner == null) { + return CompletableFuture.completedFuture(null); + } + + return brokerRegistry.lookupAsync(newOwner) + .thenApply(lookupData -> { + if (lookupData.isPresent()) { + return newOwner; + } else { + throw new IllegalStateException( + "The new owner " + newOwner + " is inactive."); + } + }); + }).whenComplete((__, e) -> { if (e != null) { - log.error("Failed to get active owner broker. serviceUnit:{}, state:{}, owner:{}", - serviceUnit, state, owner, e); + log.error("{} failed to get active owner broker. serviceUnit:{}, state:{}, owner:{}", + brokerId, serviceUnit, state, owner, e); ownerLookUpCounters.get(state).getFailure().incrementAndGet(); } - }); + }).thenApply(Optional::ofNullable); } public CompletableFuture> getOwnerAsync(String serviceUnit) { @@ -544,6 +551,25 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { } } + private Optional getOwner(String serviceUnit) { + ServiceUnitStateData data = tableview.get(serviceUnit); + ServiceUnitState state = state(data); + switch (state) { + case Owned -> { + return Optional.of(data.dstBroker()); + } + case Splitting -> { + return Optional.of(data.sourceBroker()); + } + case Init, Free -> { + return Optional.empty(); + } + default -> { + return null; + } + } + } + private long getNextVersionId(String serviceUnit) { var data = tableview.get(serviceUnit); return getNextVersionId(data); @@ -697,7 +723,7 @@ private AtomicLong getHandlerCounter(ServiceUnitStateData data, boolean total) { private void log(Throwable e, String serviceUnit, ServiceUnitStateData data, ServiceUnitStateData next) { if (e == null) { - if (log.isDebugEnabled() || isTransferCommand(data)) { + if (debug() || isTransferCommand(data)) { long handlerTotalCount = getHandlerTotalCounter(data).get(); long handlerFailureCount = getHandlerFailureCounter(data).get(); log.info("{} handled {} event for serviceUnit:{}, cur:{}, next:{}, " @@ -736,6 +762,9 @@ private void handleSkippedEvent(String serviceUnit) { private void handleOwnEvent(String serviceUnit, ServiceUnitStateData data) { var getOwnerRequest = getOwnerRequests.remove(serviceUnit); if (getOwnerRequest != null) { + if (debug()) { + log.info("Returned owner request for serviceUnit:{}", serviceUnit); + } getOwnerRequest.complete(data.dstBroker()); } stateChangeListeners.notify(serviceUnit, data, null); @@ -848,26 +877,52 @@ private boolean isTargetBroker(String broker) { } private CompletableFuture deferGetOwnerRequest(String serviceUnit) { + var requested = new MutableObject>(); try { return getOwnerRequests .computeIfAbsent(serviceUnit, k -> { - CompletableFuture future = new CompletableFuture<>(); + var ownerBefore = getOwner(serviceUnit); + if (ownerBefore != null && ownerBefore.isPresent()) { + // Here, we do a quick active check first with the computeIfAbsent lock + brokerRegistry.lookupAsync(ownerBefore.get()).getNow(Optional.empty()) + .ifPresent(__ -> requested.setValue( + CompletableFuture.completedFuture(ownerBefore.get()))); + + if (requested.getValue() != null) { + return requested.getValue(); + } + } + + + CompletableFuture future = + new CompletableFuture().orTimeout(inFlightStateWaitingTimeInMillis, + TimeUnit.MILLISECONDS) + .exceptionally(e -> { + var ownerAfter = getOwner(serviceUnit); + log.warn("{} failed to wait for owner for serviceUnit:{}; Trying to " + + "return the current owner:{}", + brokerId, serviceUnit, ownerAfter, e); + if (ownerAfter == null) { + throw new IllegalStateException(e); + } + return ownerAfter.orElse(null); + }); + if (debug()) { + log.info("{} is waiting for owner for serviceUnit:{}", brokerId, serviceUnit); + } requested.setValue(future); return future; }); } finally { var future = requested.getValue(); if (future != null) { - future.orTimeout(inFlightStateWaitingTimeInMillis + 5 * 1000, TimeUnit.MILLISECONDS) - .whenComplete((v, e) -> { - if (e != null) { - getOwnerRequests.remove(serviceUnit, future); - log.warn("Failed to getOwner for serviceUnit:{}", - serviceUnit, e); - } - } - ); + future.whenComplete((__, e) -> { + getOwnerRequests.remove(serviceUnit); + if (e != null) { + log.warn("{} failed to getOwner for serviceUnit:{}", brokerId, serviceUnit, e); + } + }); } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java index 18f39abd76b76..bcb2657c67f05 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreFactory.java @@ -18,15 +18,16 @@ */ package org.apache.pulsar.broker.loadbalance.extensions.store; -import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.broker.PulsarService; /** * The load data store factory, use to create the load data store. */ public class LoadDataStoreFactory { - public static LoadDataStore create(PulsarClient client, String name, Class clazz) + public static LoadDataStore create(PulsarService pulsar, String name, + Class clazz) throws LoadDataStoreException { - return new TableViewLoadDataStoreImpl<>(client, name, clazz); + return new TableViewLoadDataStoreImpl<>(pulsar, name, clazz); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java index 56afbef04565c..d916e91716223 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/store/TableViewLoadDataStoreImpl.java @@ -23,34 +23,46 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.TableView; -import org.apache.pulsar.common.util.FutureUtil; /** * The load data store, base on {@link TableView }. * * @param Load data type. */ +@Slf4j public class TableViewLoadDataStoreImpl implements LoadDataStore { + private static final long LOAD_DATA_REPORT_UPDATE_MAX_INTERVAL_MULTIPLIER_BEFORE_RESTART = 2; + private volatile TableView tableView; + private volatile long tableViewLastUpdateTimestamp; private volatile Producer producer; + private final ServiceConfiguration conf; + private final PulsarClient client; private final String topic; private final Class clazz; - public TableViewLoadDataStoreImpl(PulsarClient client, String topic, Class clazz) throws LoadDataStoreException { + public TableViewLoadDataStoreImpl(PulsarService pulsar, String topic, Class clazz) + throws LoadDataStoreException { try { - this.client = client; + this.conf = pulsar.getConfiguration(); + this.client = pulsar.getClient(); this.topic = topic; this.clazz = clazz; } catch (Exception e) { @@ -60,40 +72,36 @@ public TableViewLoadDataStoreImpl(PulsarClient client, String topic, Class cl @Override public synchronized CompletableFuture pushAsync(String key, T loadData) { - if (producer == null) { - return FutureUtil.failedFuture(new IllegalStateException("producer has not been started")); - } + validateProducer(); return producer.newMessage().key(key).value(loadData).sendAsync().thenAccept(__ -> {}); } @Override public synchronized CompletableFuture removeAsync(String key) { - if (producer == null) { - return FutureUtil.failedFuture(new IllegalStateException("producer has not been started")); - } + validateProducer(); return producer.newMessage().key(key).value(null).sendAsync().thenAccept(__ -> {}); } @Override public synchronized Optional get(String key) { - validateTableViewStart(); + validateTableView(); return Optional.ofNullable(tableView.get(key)); } @Override public synchronized void forEach(BiConsumer action) { - validateTableViewStart(); + validateTableView(); tableView.forEach(action); } public synchronized Set> entrySet() { - validateTableViewStart(); + validateTableView(); return tableView.entrySet(); } @Override public synchronized int size() { - validateTableViewStart(); + validateTableView(); return tableView.size(); } @@ -116,6 +124,8 @@ public synchronized void startTableView() throws LoadDataStoreException { if (tableView == null) { try { tableView = client.newTableViewBuilder(Schema.JSON(clazz)).topic(topic).create(); + tableView.forEachAndListen((k, v) -> + tableViewLastUpdateTimestamp = System.currentTimeMillis()); } catch (PulsarClientException e) { tableView = null; throw new LoadDataStoreException(e); @@ -150,9 +160,48 @@ public synchronized void init() throws IOException { start(); } - private synchronized void validateTableViewStart() { + private void validateProducer() { + if (producer == null || !producer.isConnected()) { + try { + if (producer != null) { + producer.close(); + } + producer = null; + startProducer(); + log.info("Restarted producer on {}", topic); + } catch (Exception e) { + log.error("Failed to restart producer on {}", topic, e); + throw new RuntimeException(e); + } + } + } + + private void validateTableView() { + String restartReason = null; + if (tableView == null) { - throw new IllegalStateException("table view has not been started"); + restartReason = "table view is null"; + } else { + long inactiveDuration = System.currentTimeMillis() - tableViewLastUpdateTimestamp; + long threshold = TimeUnit.MINUTES.toMillis(conf.getLoadBalancerReportUpdateMaxIntervalMinutes()) + * LOAD_DATA_REPORT_UPDATE_MAX_INTERVAL_MULTIPLIER_BEFORE_RESTART; + if (inactiveDuration > threshold) { + restartReason = String.format("inactiveDuration=%d secs > threshold = %d secs", + TimeUnit.MILLISECONDS.toSeconds(inactiveDuration), + TimeUnit.MILLISECONDS.toSeconds(threshold)); + } + } + + if (StringUtils.isNotBlank(restartReason)) { + tableViewLastUpdateTimestamp = 0; + try { + closeTableView(); + startTableView(); + log.info("Restarted tableview on {}, {}", topic, restartReason); + } catch (Exception e) { + log.error("Failed to restart tableview on {}", topic, e); + throw new RuntimeException(e); + } } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java index ba8d3db7178d9..374f1e30c0a89 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchMessageContainerImpl.java @@ -187,6 +187,7 @@ public ByteBuf toByteBuf() { idData.writeTo(buf); buf.writeInt(metadataAndPayload.readableBytes()); buf.writeBytes(metadataAndPayload); + metadataAndPayload.release(); encryptedPayload.release(); clear(); return buf; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index 4ec884a624e13..e87532cdc6ce2 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -115,6 +115,7 @@ import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.awaitility.Awaitility; import org.mockito.MockedStatic; +import org.testng.AssertJUnit; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; @@ -141,64 +142,53 @@ public class ExtensibleLoadManagerImplTest extends MockedPulsarServiceBaseTest { private final String defaultTestNamespace = "public/test"; + private static void initConfig(ServiceConfiguration conf){ + conf.setForceDeleteNamespaceAllowed(true); + conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); + conf.setAllowAutoTopicCreation(true); + conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); + conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); + conf.setLoadBalancerSheddingEnabled(false); + conf.setLoadBalancerDebugModeEnabled(true); + conf.setTopicLevelPoliciesEnabled(true); + } + @BeforeClass @Override public void setup() throws Exception { - try (MockedStatic channelMockedStatic = - mockStatic(ServiceUnitStateChannelImpl.class)) { - channelMockedStatic.when(() -> ServiceUnitStateChannelImpl.newInstance(isA(PulsarService.class))) - .thenAnswer(invocation -> { - PulsarService pulsarService = invocation.getArgument(0); - // Set the inflight state waiting time and ownership monitor delay time to 5 seconds to avoid - // stuck when doing unload. - return new ServiceUnitStateChannelImpl(pulsarService, 5 * 1000, 1); - }); - conf.setForceDeleteNamespaceAllowed(true); - conf.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); - conf.setAllowAutoTopicCreation(true); - conf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); - conf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - conf.setLoadBalancerSheddingEnabled(false); - conf.setLoadBalancerDebugModeEnabled(true); - conf.setTopicLevelPoliciesEnabled(true); - super.internalSetup(conf); - pulsar1 = pulsar; - ServiceConfiguration defaultConf = getDefaultConf(); - defaultConf.setAllowAutoTopicCreation(true); - defaultConf.setForceDeleteNamespaceAllowed(true); - defaultConf.setLoadManagerClassName(ExtensibleLoadManagerImpl.class.getName()); - defaultConf.setLoadBalancerLoadSheddingStrategy(TransferShedder.class.getName()); - defaultConf.setLoadBalancerSheddingEnabled(false); - defaultConf.setTopicLevelPoliciesEnabled(true); - additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); - pulsar2 = additionalPulsarTestContext.getPulsarService(); - - setPrimaryLoadManager(); - - setSecondaryLoadManager(); - - admin.clusters().createCluster(this.conf.getClusterName(), - ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); - admin.tenants().createTenant("public", - new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), - Sets.newHashSet(this.conf.getClusterName()))); - admin.namespaces().createNamespace("public/default"); - admin.namespaces().setNamespaceReplicationClusters("public/default", - Sets.newHashSet(this.conf.getClusterName())); - - admin.namespaces().createNamespace(defaultTestNamespace); - admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace, - Sets.newHashSet(this.conf.getClusterName())); - } + // Set the inflight state waiting time and ownership monitor delay time to 5 seconds to avoid + // stuck when doing unload. + initConfig(conf); + super.internalSetup(conf); + pulsar1 = pulsar; + ServiceConfiguration defaultConf = getDefaultConf(); + initConfig(defaultConf); + additionalPulsarTestContext = createAdditionalPulsarTestContext(defaultConf); + pulsar2 = additionalPulsarTestContext.getPulsarService(); + + setPrimaryLoadManager(); + + setSecondaryLoadManager(); + + admin.clusters().createCluster(this.conf.getClusterName(), + ClusterData.builder().serviceUrl(pulsar.getWebServiceAddress()).build()); + admin.tenants().createTenant("public", + new TenantInfoImpl(Sets.newHashSet("appid1", "appid2"), + Sets.newHashSet(this.conf.getClusterName()))); + admin.namespaces().createNamespace("public/default"); + admin.namespaces().setNamespaceReplicationClusters("public/default", + Sets.newHashSet(this.conf.getClusterName())); + + admin.namespaces().createNamespace(defaultTestNamespace); + admin.namespaces().setNamespaceReplicationClusters(defaultTestNamespace, + Sets.newHashSet(this.conf.getClusterName())); } @Override @AfterClass(alwaysRun = true) protected void cleanup() throws Exception { - pulsar1 = null; - pulsar2.close(); - super.internalCleanup(); this.additionalPulsarTestContext.close(); + super.internalCleanup(); } @BeforeMethod(alwaysRun = true) @@ -236,9 +226,6 @@ public void testAssign() throws Exception { Optional brokerLookupData1 = secondaryLoadManager.assign(Optional.empty(), bundle).get(); assertEquals(brokerLookupData, brokerLookupData1); - verify(primaryLoadManager, times(1)).getBrokerSelectionStrategy(); - verify(secondaryLoadManager, times(0)).getBrokerSelectionStrategy(); - Optional lookupResult = pulsar2.getNamespaceService() .getBrokerServiceUrlAsync(topicName, null).get(); assertTrue(lookupResult.isPresent()); @@ -462,7 +449,8 @@ public void testSplitBundleWithSpecificPositionAdminAPI() throws Exception { "specified_positions_divide", List.of(bundleRanges.get(0), bundleRanges.get(1), splitPosition)); BundlesData bundlesData = admin.namespaces().getBundles(namespace); - assertEquals(bundlesData.getNumBundles(), numBundles + 1); + Awaitility.waitAtMost(15, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals(bundlesData.getNumBundles(), numBundles + 1)); String lowBundle = String.format("0x%08x", bundleRanges.get(0)); String midBundle = String.format("0x%08x", splitPosition); String highBundle = String.format("0x%08x", bundleRanges.get(1)); @@ -475,15 +463,26 @@ public void testDeleteNamespaceBundle() throws Exception { final String namespace = "public/testDeleteNamespaceBundle"; admin.namespaces().createNamespace(namespace, 3); TopicName topicName = TopicName.get(namespace + "/test-delete-namespace-bundle"); - NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); - String broker = admin.lookups().lookupTopic(topicName.toString()); - log.info("Assign the bundle {} to {}", bundle, broker); - - checkOwnershipState(broker, bundle); - admin.namespaces().deleteNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange()); - assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + Awaitility.await() + .atMost(15, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> { + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + String broker = admin.lookups().lookupTopic(topicName.toString()); + log.info("Assign the bundle {} to {}", bundle, broker); + checkOwnershipState(broker, bundle); + admin.namespaces().deleteNamespaceBundle(topicName.getNamespace(), bundle.getBundleRange(), true); + // this could fail if the system topic lookup asynchronously happens before this. + // we will retry if it fails. + assertFalse(primaryLoadManager.checkOwnershipAsync(Optional.empty(), bundle).get()); + }); + + Awaitility.await() + .atMost(15, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> admin.namespaces().deleteNamespace(namespace, true)); } @Test(timeOut = 30 * 1000) @@ -708,7 +707,7 @@ private void assertLookupSLANamespaceOwner(PulsarService pulsar, assertEquals(result, expectedBrokerServiceUrl); } - @Test + @Test(priority = 10) public void testTopBundlesLoadDataStoreTableViewFromChannelOwner() throws Exception { var topBundlesLoadDataStorePrimary = (LoadDataStore) FieldUtils.readDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", true); @@ -1219,6 +1218,21 @@ public void testHealthcheck() throws PulsarAdminException { admin.brokers().healthcheck(TopicVersion.V2); } + @Test(timeOut = 30 * 1000) + public void compactionScheduleTest() { + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(30, TimeUnit.SECONDS) + .ignoreExceptions() + .untilAsserted(() -> { // wait until true + primaryLoadManager.monitor(); + secondaryLoadManager.monitor(); + var threshold = admin.topicPolicies() + .getCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, false); + AssertJUnit.assertEquals(5 * 1024 * 1024, threshold == null ? 0 : threshold.longValue()); + }); + } + private static abstract class MockBrokerFilter implements BrokerFilter { @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 158b91fd2773d..84988a3b3bf69 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -86,7 +86,6 @@ import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.api.Producer; -import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.TypedMessageBuilder; import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.policies.data.TopicType; @@ -331,23 +330,6 @@ private int validateChannelStart(ServiceUnitStateChannelImpl channel) return errorCnt; } - @Test(priority = 1) - public void compactionScheduleTest() { - - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> { // wait until true - try { - var threshold = admin.topicPolicies() - .getCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, false).longValue(); - assertEquals(5 * 1024 * 1024, threshold); - } catch (Exception e) { - ; - } - }); - } - @Test(priority = 2) public void assignmentTest() throws ExecutionException, InterruptedException, IllegalAccessException, TimeoutException { @@ -927,8 +909,7 @@ public void handleBrokerDeletionEventTest() } @Test(priority = 10) - public void conflictAndCompactionTest() throws ExecutionException, InterruptedException, TimeoutException, - IllegalAccessException, PulsarClientException, PulsarServerException { + public void conflictAndCompactionTest() throws Exception { String bundle = String.format("%s/%s", "public/default", "0x0000000a_0xffffffff"); var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); @@ -961,26 +942,41 @@ public void conflictAndCompactionTest() throws ExecutionException, InterruptedEx Field strategicCompactorField = FieldUtils.getDeclaredField(PulsarService.class, "strategicCompactor", true); FieldUtils.writeField(strategicCompactorField, pulsar1, compactor, true); FieldUtils.writeField(strategicCompactorField, pulsar2, compactor, true); - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(140, TimeUnit.SECONDS) - .untilAsserted(() -> { - channel1.publishAssignEventAsync(bundle, brokerId1); - verify(compactor, times(1)) - .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any()); - }); + + var threshold = admin.topicPolicies() + .getCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC); + admin.topicPolicies() + .setCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, 0); + + try { + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(140, TimeUnit.SECONDS) + .untilAsserted(() -> { + channel1.publishAssignEventAsync(bundle, brokerId1); + verify(compactor, times(1)) + .compact(eq(ServiceUnitStateChannelImpl.TOPIC), any()); + }); + + + var channel3 = createChannel(pulsar); + channel3.start(); + Awaitility.await() + .pollInterval(200, TimeUnit.MILLISECONDS) + .atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> assertEquals( + channel3.getOwnerAsync(bundle).get(), Optional.of(brokerId1))); + channel3.close(); + } finally { + FieldUtils.writeDeclaredField(channel2, + "inFlightStateWaitingTimeInMillis", 30 * 1000, true); + if (threshold != null) { + admin.topicPolicies() + .setCompactionThreshold(ServiceUnitStateChannelImpl.TOPIC, threshold); + } + } - var channel3 = createChannel(pulsar); - channel3.start(); - Awaitility.await() - .pollInterval(200, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .untilAsserted(() -> assertEquals( - channel3.getOwnerAsync(bundle).get(), Optional.of(brokerId1))); - channel3.close(); - FieldUtils.writeDeclaredField(channel2, - "inFlightStateWaitingTimeInMillis", 30 * 1000, true); } @Test(priority = 11) @@ -1583,7 +1579,7 @@ public void testActiveGetOwner() throws Exception { // verify getOwnerAsync times out because the owner is inactive now. long start = System.currentTimeMillis(); var ex = expectThrows(ExecutionException.class, () -> channel1.getOwnerAsync(bundle).get()); - assertTrue(ex.getCause() instanceof TimeoutException); + assertTrue(ex.getCause() instanceof IllegalStateException); assertTrue(System.currentTimeMillis() - start >= 1000); // simulate ownership cleanup(no selected owner) by the leader channel @@ -1783,6 +1779,8 @@ private static void overrideTableView(ServiceUnitStateChannel channel, String se throws IllegalAccessException { var tv = (TableViewImpl) FieldUtils.readField(channel, "tableview", true); + var getOwnerRequests = (Map>) + FieldUtils.readField(channel, "getOwnerRequests", true); var cache = (ConcurrentMap) FieldUtils.readField(tv, "data", true); if(val == null){ @@ -1790,6 +1788,7 @@ private static void overrideTableView(ServiceUnitStateChannel channel, String se } else { cache.put(serviceUnit, val); } + getOwnerRequests.clear(); } private static void cleanOpsCounters(ServiceUnitStateChannel channel) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java index f486370400c92..d25cba2bd1bdd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/store/LoadDataStoreTest.java @@ -20,8 +20,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.fail; import static org.testng.AssertJUnit.assertTrue; import com.google.common.collect.Sets; @@ -29,6 +27,7 @@ import lombok.Cleanup; import lombok.Data; import lombok.NoArgsConstructor; +import org.apache.commons.lang.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicDomain; @@ -40,7 +39,6 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; -import java.util.concurrent.ExecutionException; @Test(groups = "broker") public class LoadDataStoreTest extends MockedPulsarServiceBaseTest { @@ -76,7 +74,7 @@ public void testPushGetAndRemove() throws Exception { @Cleanup LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, MyClass.class); + LoadDataStoreFactory.create(pulsar, topic, MyClass.class); loadDataStore.startProducer(); loadDataStore.startTableView(); MyClass myClass1 = new MyClass("1", 1); @@ -110,7 +108,7 @@ public void testForEach() throws Exception { @Cleanup LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class); + LoadDataStoreFactory.create(pulsar, topic, Integer.class); loadDataStore.startProducer(); loadDataStore.startTableView(); @@ -135,7 +133,7 @@ public void testForEach() throws Exception { public void testTableViewRestart() throws Exception { String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class); + LoadDataStoreFactory.create(pulsar, topic, Integer.class); loadDataStore.startProducer(); loadDataStore.startTableView(); @@ -145,43 +143,26 @@ public void testTableViewRestart() throws Exception { loadDataStore.closeTableView(); loadDataStore.pushAsync("1", 2).get(); - Exception ex = null; - try { - loadDataStore.get("1"); - } catch (IllegalStateException e) { - ex = e; - } - assertNotNull(ex); - loadDataStore.startTableView(); Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 2)); + + loadDataStore.pushAsync("1", 3).get(); + FieldUtils.writeField(loadDataStore, "tableViewLastUpdateTimestamp", 0 , true); + Awaitility.await().untilAsserted(() -> assertEquals(loadDataStore.get("1").get(), 3)); } @Test public void testProducerStop() throws Exception { String topic = TopicDomain.persistent + "://" + NamespaceName.SYSTEM_NAMESPACE + "/" + UUID.randomUUID(); LoadDataStore loadDataStore = - LoadDataStoreFactory.create(pulsar.getClient(), topic, Integer.class); + LoadDataStoreFactory.create(pulsar, topic, Integer.class); loadDataStore.startProducer(); loadDataStore.pushAsync("1", 1).get(); loadDataStore.removeAsync("1").get(); loadDataStore.close(); - try { - loadDataStore.pushAsync("2", 2).get(); - fail(); - } catch (ExecutionException ex) { - assertTrue(ex.getCause() instanceof IllegalStateException); - } - try { - loadDataStore.removeAsync("2").get(); - fail(); - } catch (ExecutionException ex) { - assertTrue(ex.getCause() instanceof IllegalStateException); - } - loadDataStore.startProducer(); - loadDataStore.pushAsync("3", 3).get(); - loadDataStore.removeAsync("3").get(); + loadDataStore.pushAsync("2", 2).get(); + loadDataStore.removeAsync("2").get(); } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java index 560636f94622b..d46f7fa140887 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TableViewImpl.java @@ -284,8 +284,14 @@ private void readTailMessages(Reader reader) { log.error("Reader {} was closed while reading tail messages.", reader.getTopic(), ex); } else { + // Retrying on the other exceptions such as NotConnectedException + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } log.warn("Reader {} was interrupted while reading tail messages. " - + "Retrying..", reader.getTopic(), ex); + + "Retrying..", reader.getTopic(), ex); readTailMessages(reader); } return null; diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index e262b27fe2306..b9707ea76c352 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -91,7 +91,6 @@ public void setup() throws Exception { "org.apache.pulsar.broker.loadbalance.extensions.scheduler.TransferShedder"); brokerEnvs.put("forceDeleteNamespaceAllowed", "true"); brokerEnvs.put("loadBalancerDebugModeEnabled", "true"); - brokerEnvs.put("topicLevelPoliciesEnabled", "false"); brokerEnvs.put("PULSAR_MEM", "-Xmx512M"); spec.brokerEnvs(brokerEnvs); pulsarCluster = PulsarCluster.forSpec(spec); From b3b1bfb3e2a29674cc9d6144baeef1a3f0058c07 Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Wed, 28 Feb 2024 18:18:22 -0800 Subject: [PATCH 485/494] [fix][broker][branch-3.0] Return getOwnerAsync without waiting on source broker upon Assigning and Releasing and handle role change during role init (#22112) (#22156) --- .../extensions/ExtensibleLoadManagerImpl.java | 24 ++++ .../channel/ServiceUnitStateChannelImpl.java | 11 +- .../ExtensibleLoadManagerImplTest.java | 128 ++++++++++++++---- .../channel/ServiceUnitStateChannelTest.java | 77 +++++++++-- .../ExtensibleLoadManagerTest.java | 35 +++-- 5 files changed, 224 insertions(+), 51 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java index 409bb55075be0..6a0e677c66268 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImpl.java @@ -777,8 +777,13 @@ synchronized void playLeader() { log.info("This broker:{} is setting the role from {} to {}", pulsar.getBrokerId(), role, Leader); int retry = 0; + boolean becameFollower = false; while (!Thread.currentThread().isInterrupted()) { try { + if (!serviceUnitStateChannel.isChannelOwner()) { + becameFollower = true; + break; + } initWaiter.await(); // Confirm the system topics have been created or create them if they do not exist. // If the leader has changed, the new leader need to reset @@ -802,6 +807,13 @@ synchronized void playLeader() { } } } + + if (becameFollower) { + log.warn("The broker:{} became follower while initializing leader role.", pulsar.getBrokerId()); + playFollower(); + return; + } + role = Leader; log.info("This broker:{} plays the leader now.", pulsar.getBrokerId()); @@ -815,8 +827,13 @@ synchronized void playFollower() { log.info("This broker:{} is setting the role from {} to {}", pulsar.getBrokerId(), role, Follower); int retry = 0; + boolean becameLeader = false; while (!Thread.currentThread().isInterrupted()) { try { + if (serviceUnitStateChannel.isChannelOwner()) { + becameLeader = true; + break; + } initWaiter.await(); unloadScheduler.close(); serviceUnitStateChannel.cancelOwnershipMonitor(); @@ -836,6 +853,13 @@ synchronized void playFollower() { } } } + + if (becameLeader) { + log.warn("This broker:{} became leader while initializing follower role.", pulsar.getBrokerId()); + playLeader(); + return; + } + role = Follower; log.info("This broker:{} plays a follower now.", pulsar.getBrokerId()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java index 220ce02ba5aef..f66ed2a5c9062 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelImpl.java @@ -533,7 +533,16 @@ public CompletableFuture> getOwnerAsync(String serviceUnit) { return getActiveOwnerAsync(serviceUnit, state, Optional.of(data.sourceBroker())); } case Assigning, Releasing -> { - return getActiveOwnerAsync(serviceUnit, state, Optional.empty()); + if (isTargetBroker(data.dstBroker())) { + return getActiveOwnerAsync(serviceUnit, state, Optional.of(data.dstBroker())); + } + // If this broker is not the dst broker, return the dst broker as the owner(or empty). + // Clients need to connect(redirect) to the dst broker anyway + // and wait for the dst broker to receive `Owned`. + // This is also required to return getOwnerAsync on the src broker immediately during unloading. + // Otherwise, topic creation(getOwnerAsync) could block unloading bundles, + // if the topic creation(getOwnerAsync) happens during unloading on the src broker. + return CompletableFuture.completedFuture(Optional.ofNullable(data.dstBroker())); } case Init, Free -> { return CompletableFuture.completedFuture(Optional.empty()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java index e87532cdc6ce2..43b84a5426f3c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplTest.java @@ -18,6 +18,8 @@ */ package org.apache.pulsar.broker.loadbalance.extensions; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState.Releasing; +import static org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelTest.overrideTableView; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Admin; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.Bandwidth; import static org.apache.pulsar.broker.loadbalance.extensions.models.SplitDecision.Reason.MsgRate; @@ -70,6 +72,7 @@ import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -78,6 +81,7 @@ import org.apache.pulsar.broker.loadbalance.LeaderElectionService; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitState; import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateChannelImpl; +import org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLoadData; import org.apache.pulsar.broker.loadbalance.extensions.data.BrokerLookupData; import org.apache.pulsar.broker.loadbalance.extensions.data.TopBundlesLoadData; @@ -95,12 +99,14 @@ import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.broker.service.BrokerServiceException; import org.apache.pulsar.broker.testcontext.PulsarTestContext; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.impl.TableViewImpl; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.naming.SystemTopicNames; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.naming.TopicVersion; import org.apache.pulsar.common.policies.data.BrokerAssignment; @@ -775,7 +781,6 @@ public void testRoleChange() throws Exception { reset(); return null; }).when(topBundlesLoadDataStorePrimarySpy).closeTableView(); - FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimarySpy, true); var topBundlesLoadDataStoreSecondary = (LoadDataStore) FieldUtils.readDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", true); @@ -798,36 +803,65 @@ public void testRoleChange() throws Exception { reset(); return null; }).when(topBundlesLoadDataStoreSecondarySpy).closeTableView(); - FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondarySpy, true); - if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { - primaryLoadManager.playFollower(); // close 3 times - primaryLoadManager.playFollower(); // close 1 time - secondaryLoadManager.playLeader(); - secondaryLoadManager.playLeader(); - primaryLoadManager.playLeader(); // close 3 times and open 3 times - primaryLoadManager.playLeader(); // close 1 time and open 1 time, - secondaryLoadManager.playFollower(); - secondaryLoadManager.playFollower(); - } else { - primaryLoadManager.playLeader(); - primaryLoadManager.playLeader(); - secondaryLoadManager.playFollower(); - secondaryLoadManager.playFollower(); - primaryLoadManager.playFollower(); + try { + FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStorePrimarySpy, true); + FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStoreSecondarySpy, true); + + + if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { + primaryLoadManager.playLeader(); + secondaryLoadManager.playFollower(); + verify(topBundlesLoadDataStorePrimarySpy, times(3)).startTableView(); + verify(topBundlesLoadDataStorePrimarySpy, times(5)).closeTableView(); + verify(topBundlesLoadDataStoreSecondarySpy, times(0)).startTableView(); + verify(topBundlesLoadDataStoreSecondarySpy, times(3)).closeTableView(); + } else { + primaryLoadManager.playFollower(); + secondaryLoadManager.playLeader(); + verify(topBundlesLoadDataStoreSecondarySpy, times(3)).startTableView(); + verify(topBundlesLoadDataStoreSecondarySpy, times(5)).closeTableView(); + verify(topBundlesLoadDataStorePrimarySpy, times(0)).startTableView(); + verify(topBundlesLoadDataStorePrimarySpy, times(3)).closeTableView(); + } + primaryLoadManager.playFollower(); - secondaryLoadManager.playLeader(); - secondaryLoadManager.playLeader(); - } + secondaryLoadManager.playFollower(); + if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + FieldUtils.readDeclaredField(primaryLoadManager, "role", true)); + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + FieldUtils.readDeclaredField(secondaryLoadManager, "role", true)); + } else { + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + FieldUtils.readDeclaredField(primaryLoadManager, "role", true)); + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + FieldUtils.readDeclaredField(secondaryLoadManager, "role", true)); + } - verify(topBundlesLoadDataStorePrimarySpy, times(4)).startTableView(); - verify(topBundlesLoadDataStorePrimarySpy, times(8)).closeTableView(); - verify(topBundlesLoadDataStoreSecondarySpy, times(4)).startTableView(); - verify(topBundlesLoadDataStoreSecondarySpy, times(8)).closeTableView(); + primaryLoadManager.playLeader(); + secondaryLoadManager.playLeader(); - FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStorePrimary, true); - FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", topBundlesLoadDataStoreSecondary, true); + if (channel1.isChannelOwnerAsync().get(5, TimeUnit.SECONDS)) { + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + FieldUtils.readDeclaredField(primaryLoadManager, "role", true)); + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + FieldUtils.readDeclaredField(secondaryLoadManager, "role", true)); + } else { + assertEquals(ExtensibleLoadManagerImpl.Role.Follower, + FieldUtils.readDeclaredField(primaryLoadManager, "role", true)); + assertEquals(ExtensibleLoadManagerImpl.Role.Leader, + FieldUtils.readDeclaredField(secondaryLoadManager, "role", true)); + } + } finally { + FieldUtils.writeDeclaredField(primaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStorePrimary, true); + FieldUtils.writeDeclaredField(secondaryLoadManager, "topBundlesLoadDataStore", + topBundlesLoadDataStoreSecondary, true); + } } @Test @@ -1233,6 +1267,32 @@ public void compactionScheduleTest() { }); } + @Test(timeOut = 10 * 1000) + public void unloadTimeoutCheckTest() + throws Exception { + Pair topicAndBundle = getBundleIsNotOwnByChangeEventTopic("unload-timeout"); + String topic = topicAndBundle.getLeft().toString(); + var bundle = topicAndBundle.getRight().toString(); + var releasing = new ServiceUnitStateData(Releasing, pulsar2.getBrokerId(), pulsar1.getBrokerId(), 1); + overrideTableView(channel1, bundle, releasing); + var topicFuture = pulsar1.getBrokerService().getOrCreateTopic(topic); + + + try { + topicFuture.get(1, TimeUnit.SECONDS); + } catch (Exception e) { + log.info("getOrCreateTopic failed", e); + if (!(e.getCause() instanceof BrokerServiceException.ServiceUnitNotReadyException && e.getMessage() + .contains("Please redo the lookup"))) { + fail(); + } + } + + pulsar1.getBrokerService() + .unloadServiceUnit(topicAndBundle.getRight(), true, 5, + TimeUnit.SECONDS).get(2, TimeUnit.SECONDS); + } + private static abstract class MockBrokerFilter implements BrokerFilter { @Override @@ -1265,4 +1325,20 @@ private void setSecondaryLoadManager() throws IllegalAccessException { private CompletableFuture getBundleAsync(PulsarService pulsar, TopicName topic) { return pulsar.getNamespaceService().getBundleAsync(topic); } + + private Pair getBundleIsNotOwnByChangeEventTopic(String topicNamePrefix) + throws Exception { + TopicName changeEventsTopicName = + TopicName.get(defaultTestNamespace + "/" + SystemTopicNames.NAMESPACE_EVENTS_LOCAL_NAME); + NamespaceBundle changeEventsBundle = getBundleAsync(pulsar1, changeEventsTopicName).get(); + int i = 0; + while (true) { + TopicName topicName = TopicName.get(defaultTestNamespace + "/" + topicNamePrefix + "-" + i); + NamespaceBundle bundle = getBundleAsync(pulsar1, topicName).get(); + if (!bundle.equals(changeEventsBundle)) { + return Pair.of(topicName, bundle); + } + i++; + } + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java index 84988a3b3bf69..dc9844366771c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/channel/ServiceUnitStateChannelTest.java @@ -484,19 +484,17 @@ public void transferTestWhenDestBrokerFails() var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); - assertFalse(owner1.isDone()); + assertTrue(owner1.isDone()); + assertEquals(brokerId2, owner1.get().get()); assertFalse(owner2.isDone()); - assertEquals(1, getOwnerRequests1.size()); + assertEquals(0, getOwnerRequests1.size()); assertEquals(1, getOwnerRequests2.size()); // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. - Awaitility.await().atMost(10, TimeUnit.SECONDS) - .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); - assertEquals(0, getOwnerRequests1.size()); assertEquals(0, getOwnerRequests2.size()); // recovered, check the monitor update state : Assigned -> Owned @@ -1133,12 +1131,10 @@ public void assignTestWhenDestBrokerProducerFails() var owner1 = channel1.getOwnerAsync(bundle); var owner2 = channel2.getOwnerAsync(bundle); - assertFalse(owner1.isDone()); + assertTrue(owner1.isDone()); assertFalse(owner2.isDone()); // In 10 secs, the getOwnerAsync requests(lookup requests) should time out. - Awaitility.await().atMost(10, TimeUnit.SECONDS) - .untilAsserted(() -> assertTrue(owner1.isCompletedExceptionally())); Awaitility.await().atMost(10, TimeUnit.SECONDS) .untilAsserted(() -> assertTrue(owner2.isCompletedExceptionally())); @@ -1317,6 +1313,68 @@ public void testIsOwner() throws IllegalAccessException { assertFalse(channel1.isOwner(bundle)); } + @Test(priority = 15) + public void testGetOwnerAsync() throws Exception { + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, brokerId1, 1)); + var owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId1, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Owned, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(!owner.isDone()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Assigning, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(!owner.isDone()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Releasing, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId1, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Splitting, null, brokerId2, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(brokerId2, owner.get().get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Free, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + + overrideTableView(channel1, bundle, new ServiceUnitStateData(Deleted, null, brokerId1, 1)); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertTrue(owner.isCompletedExceptionally()); + + overrideTableView(channel1, bundle, null); + owner = channel1.getOwnerAsync(bundle); + assertTrue(owner.isDone()); + assertEquals(Optional.empty(), owner.get()); + } + @Test(priority = 16) public void splitAndRetryFailureTest() throws Exception { channel1.publishAssignEventAsync(bundle3, brokerId1); @@ -1775,7 +1833,8 @@ private void overrideTableViews(String serviceUnit, ServiceUnitStateData val) th overrideTableView(channel2, serviceUnit, val); } - private static void overrideTableView(ServiceUnitStateChannel channel, String serviceUnit, ServiceUnitStateData val) + @Test(enabled = false) + public static void overrideTableView(ServiceUnitStateChannel channel, String serviceUnit, ServiceUnitStateData val) throws IllegalAccessException { var tv = (TableViewImpl) FieldUtils.readField(channel, "tableview", true); diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java index b9707ea76c352..af14ef97f85c3 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/loadbalance/ExtensibleLoadManagerTest.java @@ -324,8 +324,8 @@ public void testAntiaffinityPolicy() throws PulsarAdminException { assertEquals(result.size(), NUM_BROKERS); } - @Test(timeOut = 40 * 1000) - public void testIsolationPolicy() throws PulsarAdminException { + @Test(timeOut = 300 * 1000) + public void testIsolationPolicy() throws Exception { final String namespaceIsolationPolicyName = "my-isolation-policy"; final String isolationEnabledNameSpace = DEFAULT_TENANT + "/my-isolation-policy" + nsSuffix; Map parameters1 = new HashMap<>(); @@ -334,7 +334,8 @@ public void testIsolationPolicy() throws PulsarAdminException { Awaitility.await().atMost(10, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( () -> { - List activeBrokers = admin.brokers().getActiveBrokers(); + List activeBrokers = admin.brokers().getActiveBrokersAsync() + .get(5, TimeUnit.SECONDS); assertEquals(activeBrokers.size(), NUM_BROKERS); } ); @@ -377,15 +378,16 @@ public void testIsolationPolicy() throws PulsarAdminException { } } - Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( () -> { - List activeBrokers = admin.brokers().getActiveBrokers(); + List activeBrokers = admin.brokers().getActiveBrokersAsync() + .get(5, TimeUnit.SECONDS); assertEquals(activeBrokers.size(), 2); } ); Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted(() -> { - String ownerBroker = admin.lookups().lookupTopic(topic); + String ownerBroker = admin.lookups().lookupTopicAsync(topic).get(5, TimeUnit.SECONDS); assertEquals(extractBrokerIndex(ownerBroker), 1); }); @@ -396,20 +398,23 @@ public void testIsolationPolicy() throws PulsarAdminException { } } - Awaitility.await().atMost(30, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( () -> { - List activeBrokers = admin.brokers().getActiveBrokers(); + List activeBrokers = admin.brokers().getActiveBrokersAsync().get(5, TimeUnit.SECONDS); assertEquals(activeBrokers.size(), 1); } ); - try { - admin.lookups().lookupTopic(topic); - fail(); - } catch (Exception ex) { - log.error("Failed to lookup topic: ", ex); - assertThat(ex.getMessage()).contains("Failed to select the new owner broker for bundle"); - } + Awaitility.await().atMost(60, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> { + try { + admin.lookups().lookupTopicAsync(topic).get(5, TimeUnit.SECONDS); + } catch (Exception ex) { + log.error("Failed to lookup topic: ", ex); + assertThat(ex.getMessage()).contains("Failed to select the new owner broker for bundle"); + } + } + ); } private String getBrokerUrl(int index) { From ca35b2eb4900b9b06ea6fd1363862fd36f615785 Mon Sep 17 00:00:00 2001 From: Heesung Sohn Date: Thu, 29 Feb 2024 11:41:26 -0800 Subject: [PATCH 486/494] Revert "[improve][admin] Expose the offload threshold in seconds to the amdin (#22101)" This reverts commit 6e988ef347ffdf31e77bc20882d84c4b6eb74adc. --- .../org/apache/pulsar/admin/cli/CmdNamespaces.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 28e8fb8d5f5b2..998591f8177d1 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1962,8 +1962,7 @@ private class GetOffloadThreshold extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - print("offloadThresholdInBytes: " + getAdmin().namespaces().getOffloadThreshold(namespace)); - print("offloadThresholdInSeconds: " + getAdmin().namespaces().getOffloadThresholdInSeconds(namespace)); + print(getAdmin().namespaces().getOffloadThreshold(namespace)); } } @@ -1981,18 +1980,11 @@ private class SetOffloadThreshold extends CliCommand { required = true) private String thresholdStr = "-1"; - @Parameter(names = {"--time", "-t"}, - description = "Maximum number of seconds stored on the pulsar cluster for a topic" - + " before the broker will start offloading to longterm storage (eg: 10m, 5h, 3d, 2w).", - converter = TimeUnitToSecondsConverter.class) - private Long thresholdInSeconds = -1L; - @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); long threshold = validateSizeString(thresholdStr); getAdmin().namespaces().setOffloadThreshold(namespace, threshold); - getAdmin().namespaces().setOffloadThresholdInSeconds(namespace, thresholdInSeconds); } } From bc13b447455b45cf75bf44c28c0bf48a20f98a95 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Thu, 29 Feb 2024 12:47:54 +0800 Subject: [PATCH 487/494] [fix] [broker] print non log when delete partitioned topic failed (#22153) --- .../pulsar/broker/service/persistent/PersistentTopic.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 3f75c02d48d4b..47e314bb3198b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -2674,7 +2674,7 @@ public void checkGC() { replCloseFuture.thenCompose(v -> delete(deleteMode == InactiveTopicDeleteMode.delete_when_no_subscriptions, deleteMode == InactiveTopicDeleteMode.delete_when_subscriptions_caught_up, false)) - .thenApply((res) -> tryToDeletePartitionedMetadata()) + .thenCompose((res) -> tryToDeletePartitionedMetadata()) .thenRun(() -> log.info("[{}] Topic deleted successfully due to inactivity", topic)) .exceptionally(e -> { if (e.getCause() instanceof TopicBusyException) { @@ -2682,6 +2682,8 @@ public void checkGC() { if (log.isDebugEnabled()) { log.debug("[{}] Did not delete busy topic: {}", topic, e.getCause().getMessage()); } + } else if (e.getCause() instanceof UnsupportedOperationException) { + log.info("[{}] Skip to delete partitioned topic: {}", topic, e.getCause().getMessage()); } else { log.warn("[{}] Inactive topic deletion failed", topic, e); } @@ -2726,7 +2728,7 @@ private CompletableFuture tryToDeletePartitionedMetadata() { .filter(topicExist -> topicExist) .findAny(); if (anyExistPartition.isPresent()) { - log.error("[{}] Delete topic metadata failed because" + log.info("[{}] Delete topic metadata failed because" + " another partition exist.", topicName); throw new UnsupportedOperationException( String.format("Another partition exists for [%s].", From 9292b05d22ee824dfc5d54d9d8a46267621f8c91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=93=E5=90=9B?= Date: Thu, 29 Feb 2024 21:03:47 +0800 Subject: [PATCH 488/494] [fix][offload] Fix Offload readHandle cannot close multi times. (#22162) (cherry picked from commit e25c7f045753b949c5ecd492bd7b7a77440c6937) --- .../impl/FileStoreBackedReadHandleImpl.java | 36 ++++++++++++++----- .../impl/BlobStoreBackedReadHandleImpl.java | 32 ++++++++++------- .../impl/BlobStoreBackedReadHandleImplV2.java | 14 ++++++-- 3 files changed, 59 insertions(+), 23 deletions(-) diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java index 506fbb8de68bf..e31c62b2bd3c8 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileStoreBackedReadHandleImpl.java @@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.api.LastConfirmedAndEntry; import org.apache.bookkeeper.client.api.LedgerEntries; @@ -37,6 +38,7 @@ import org.apache.bookkeeper.client.impl.LedgerEntriesImpl; import org.apache.bookkeeper.client.impl.LedgerEntryImpl; import org.apache.bookkeeper.mledger.LedgerOffloaderStats; +import org.apache.bookkeeper.mledger.ManagedLedgerException; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapFile; @@ -53,6 +55,12 @@ public class FileStoreBackedReadHandleImpl implements ReadHandle { private final LedgerOffloaderStats offloaderStats; private final String managedLedgerName; private final String topicName; + enum State { + Opened, + Closed + } + private volatile State state; + private final AtomicReference> closeFuture = new AtomicReference<>(); private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader reader, long ledgerId, LedgerOffloaderStats offloaderStats, @@ -72,6 +80,7 @@ private FileStoreBackedReadHandleImpl(ExecutorService executor, MapFile.Reader r offloaderStats.recordReadOffloadIndexLatency(topicName, System.nanoTime() - startReadIndexTime, TimeUnit.NANOSECONDS); this.ledgerMetadata = parseLedgerMetadata(ledgerId, value.copyBytes()); + state = State.Opened; } catch (IOException e) { log.error("Fail to read LedgerMetadata for ledgerId {}", ledgerId); @@ -92,15 +101,20 @@ public LedgerMetadata getLedgerMetadata() { @Override public CompletableFuture closeAsync() { - CompletableFuture promise = new CompletableFuture<>(); + if (closeFuture.get() != null || !closeFuture.compareAndSet(null, new CompletableFuture<>())) { + return closeFuture.get(); + } + + CompletableFuture promise = closeFuture.get(); executor.execute(() -> { - try { - reader.close(); - promise.complete(null); - } catch (IOException t) { - promise.completeExceptionally(t); - } - }); + try { + reader.close(); + state = State.Closed; + promise.complete(null); + } catch (IOException t) { + promise.completeExceptionally(t); + } + }); return promise; } @@ -111,6 +125,12 @@ public CompletableFuture readAsync(long firstEntry, long lastEntr } CompletableFuture promise = new CompletableFuture<>(); executor.execute(() -> { + if (state == State.Closed) { + log.warn("Reading a closed read handler. Ledger ID: {}, Read range: {}-{}", + ledgerId, firstEntry, lastEntry); + promise.completeExceptionally(new ManagedLedgerException.OffloadReadHandleClosedException()); + return; + } if (firstEntry > lastEntry || firstEntry < 0 || lastEntry > getLastAddConfirmed()) { diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java index 5a571bb208e34..5346be6a044c8 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImpl.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.offload.jcloud.impl; +import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import io.netty.buffer.ByteBuf; @@ -30,6 +31,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.api.LastConfirmedAndEntry; import org.apache.bookkeeper.client.api.LedgerEntries; @@ -66,13 +68,14 @@ public class BlobStoreBackedReadHandleImpl implements ReadHandle { .newBuilder() .expireAfterAccess(CACHE_TTL_SECONDS, TimeUnit.SECONDS) .build(); + private final AtomicReference> closeFuture = new AtomicReference<>(); enum State { Opened, Closed } - private State state = null; + private volatile State state = null; private BlobStoreBackedReadHandleImpl(long ledgerId, OffloadIndexBlock index, BackedInputStream inputStream, ExecutorService executor) { @@ -96,18 +99,22 @@ public LedgerMetadata getLedgerMetadata() { @Override public CompletableFuture closeAsync() { - CompletableFuture promise = new CompletableFuture<>(); + if (closeFuture.get() != null || !closeFuture.compareAndSet(null, new CompletableFuture<>())) { + return closeFuture.get(); + } + + CompletableFuture promise = closeFuture.get(); executor.execute(() -> { - try { - index.close(); - inputStream.close(); - entryOffsets.invalidateAll(); - state = State.Closed; - promise.complete(null); - } catch (IOException t) { - promise.completeExceptionally(t); - } - }); + try { + index.close(); + inputStream.close(); + entryOffsets.invalidateAll(); + state = State.Closed; + promise.complete(null); + } catch (IOException t) { + promise.completeExceptionally(t); + } + }); return promise; } @@ -298,6 +305,7 @@ public static ReadHandle open(ScheduledExecutorService executor, } // for testing + @VisibleForTesting State getState() { return this.state; } diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java index e40a0a3834c85..53d96e08abf5e 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreBackedReadHandleImplV2.java @@ -30,6 +30,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import lombok.val; import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.api.LastConfirmedAndEntry; @@ -60,7 +61,8 @@ public class BlobStoreBackedReadHandleImplV2 implements ReadHandle { private final List inputStreams; private final List dataStreams; private final ExecutorService executor; - private State state = null; + private volatile State state = null; + private final AtomicReference> closeFuture = new AtomicReference<>(); enum State { Opened, @@ -123,7 +125,11 @@ public LedgerMetadata getLedgerMetadata() { @Override public CompletableFuture closeAsync() { - CompletableFuture promise = new CompletableFuture<>(); + if (closeFuture.get() != null || !closeFuture.compareAndSet(null, new CompletableFuture<>())) { + return closeFuture.get(); + } + + CompletableFuture promise = closeFuture.get(); executor.execute(() -> { try { for (OffloadIndexBlockV2 indexBlock : indices) { @@ -143,7 +149,9 @@ public CompletableFuture closeAsync() { @Override public CompletableFuture readAsync(long firstEntry, long lastEntry) { - log.debug("Ledger {}: reading {} - {}", getId(), firstEntry, lastEntry); + if (log.isDebugEnabled()) { + log.debug("Ledger {}: reading {} - {}", getId(), firstEntry, lastEntry); + } CompletableFuture promise = new CompletableFuture<>(); executor.execute(() -> { if (state == State.Closed) { From 2aa359778eefbc47395ef3d79a52ab42f1f95d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=81=93=E5=90=9B?= Date: Thu, 29 Feb 2024 21:22:03 +0800 Subject: [PATCH 489/494] [fix][txn]Fix TopicTransactionBuffer potential thread safety issue (#22149) (cherry picked from commit 74be3fd4917a2327f2da9b5b55cc572b3c1f4e84) --- .../buffer/impl/TopicTransactionBuffer.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java index 8a76362f5a35f..b5e4b6d0f901a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/transaction/buffer/impl/TopicTransactionBuffer.java @@ -166,13 +166,15 @@ public void handleTxnEntry(Entry entry) { if (msgMetadata != null && msgMetadata.hasTxnidMostBits() && msgMetadata.hasTxnidLeastBits()) { TxnID txnID = new TxnID(msgMetadata.getTxnidMostBits(), msgMetadata.getTxnidLeastBits()); PositionImpl position = PositionImpl.get(entry.getLedgerId(), entry.getEntryId()); - if (Markers.isTxnMarker(msgMetadata)) { - if (Markers.isTxnAbortMarker(msgMetadata)) { - snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, position); + synchronized (TopicTransactionBuffer.this) { + if (Markers.isTxnMarker(msgMetadata)) { + if (Markers.isTxnAbortMarker(msgMetadata)) { + snapshotAbortedTxnProcessor.putAbortedTxnAndPosition(txnID, position); + } + updateMaxReadPosition(txnID); + } else { + handleTransactionMessage(txnID, position); } - updateMaxReadPosition(txnID); - } else { - handleTransactionMessage(txnID, position); } } } @@ -358,10 +360,10 @@ public void addComplete(Position position, ByteBuf entryData, Object ctx) { updateMaxReadPosition(txnID); snapshotAbortedTxnProcessor.trimExpiredAbortedTxns(); takeSnapshotByChangeTimes(); + txnAbortedCounter.increment(); + completableFuture.complete(null); + handleLowWaterMark(txnID, lowWaterMark); } - txnAbortedCounter.increment(); - completableFuture.complete(null); - handleLowWaterMark(txnID, lowWaterMark); } @Override @@ -470,7 +472,7 @@ public CompletableFuture closeAsync() { } @Override - public boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { + public synchronized boolean isTxnAborted(TxnID txnID, PositionImpl readPosition) { return snapshotAbortedTxnProcessor.checkAbortedTransaction(txnID); } @@ -502,9 +504,11 @@ public PositionImpl getMaxReadPosition() { @Override public TransactionInBufferStats getTransactionInBufferStats(TxnID txnID) { TransactionInBufferStats transactionInBufferStats = new TransactionInBufferStats(); - transactionInBufferStats.aborted = isTxnAborted(txnID, null); - if (ongoingTxns.containsKey(txnID)) { - transactionInBufferStats.startPosition = ongoingTxns.get(txnID).toString(); + synchronized (this) { + transactionInBufferStats.aborted = isTxnAborted(txnID, null); + if (ongoingTxns.containsKey(txnID)) { + transactionInBufferStats.startPosition = ongoingTxns.get(txnID).toString(); + } } return transactionInBufferStats; } From 624f685e3a7802a9b9f5f15fa6eb4fdc9947321d Mon Sep 17 00:00:00 2001 From: Heesung Sohn <103456639+heesung-sn@users.noreply.github.com> Date: Thu, 29 Feb 2024 15:20:22 -0800 Subject: [PATCH 490/494] Minor Compile fix (#22167) --- .../broker/service/SystemTopicBasedTopicPoliciesServiceTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index cde41ffae6835..343a18da6d8a9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -37,6 +37,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; From 21660bdf8a1c853404dc54c73bad9b50da68908b Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Sat, 2 Mar 2024 04:06:42 +0800 Subject: [PATCH 491/494] [improve][admin][branch-3.0] Expose the offload threshold in seconds to the admin (#22169) --- .../org/apache/pulsar/admin/cli/CmdNamespaces.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 998591f8177d1..9c32041a36503 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -1962,7 +1962,8 @@ private class GetOffloadThreshold extends CliCommand { @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); - print(getAdmin().namespaces().getOffloadThreshold(namespace)); + print("offloadThresholdInBytes: " + getAdmin().namespaces().getOffloadThreshold(namespace)); + print("offloadThresholdInSeconds: " + getAdmin().namespaces().getOffloadThresholdInSeconds(namespace)); } } @@ -1980,11 +1981,18 @@ private class SetOffloadThreshold extends CliCommand { required = true) private String thresholdStr = "-1"; + @Parameter(names = {"--time", "-t"}, + description = "Maximum number of seconds stored on the pulsar cluster for a topic" + + " before the broker will start offloading to longterm storage (eg: 10m, 5h, 3d, 2w).") + private String thresholdInSeconds = "-1"; + @Override void run() throws PulsarAdminException { String namespace = validateNamespace(params); long threshold = validateSizeString(thresholdStr); + long timeInSeconds = RelativeTimeUtil.parseRelativeTimeInSeconds(thresholdInSeconds); getAdmin().namespaces().setOffloadThreshold(namespace, threshold); + getAdmin().namespaces().setOffloadThresholdInSeconds(namespace, timeInSeconds); } } From f0da29b64007eaf21e83a5e9946558333035cfef Mon Sep 17 00:00:00 2001 From: Qiang Zhao Date: Thu, 29 Feb 2024 18:57:03 +0800 Subject: [PATCH 492/494] [improve][broker] Add fine-grain authorization to retention admin API (#22163) (cherry picked from commit 6ec473ed6458cf30e1fc7062057a50bfefada6cf) --- .../broker/admin/v2/PersistentTopics.java | 9 +- .../broker/admin/TopicPoliciesAuthZTest.java | 174 ++++++++++++++++++ .../security/MockedPulsarStandalone.java | 155 ++++++++++++++++ 3 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index eee363aeed86b..15d23d6e4d85f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -2406,7 +2406,8 @@ public void getRetention(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.READ) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalGetRetention(applied, isGlobal)) .thenAccept(asyncResponse::resume) .exceptionally(ex -> { @@ -2433,7 +2434,8 @@ public void setRetention(@Suspended final AsyncResponse asyncResponse, @QueryParam("isGlobal") @DefaultValue("false") boolean isGlobal, @ApiParam(value = "Retention policies for the specified topic") RetentionPolicies retention) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalSetRetention(retention, isGlobal)) .thenRun(() -> { try { @@ -2469,7 +2471,8 @@ public void removeRetention(@Suspended final AsyncResponse asyncResponse, @ApiParam(value = "Whether leader broker redirected this call to this broker. For internal use.") @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateTopicName(tenant, namespace, encodedTopic); - preValidation(authoritative) + validateTopicPolicyOperationAsync(topicName, PolicyName.RETENTION, PolicyOperation.WRITE) + .thenCompose(__ -> preValidation(authoritative)) .thenCompose(__ -> internalRemoveRetention(isGlobal)) .thenRun(() -> { log.info("[{}] Successfully remove retention: namespace={}, topic={}", diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java new file mode 100644 index 0000000000000..9cf4c111dddbc --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesAuthZTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin; + +import static org.awaitility.Awaitility.await; +import io.jsonwebtoken.Jwts; +import java.util.Set; +import java.util.UUID; +import lombok.Cleanup; +import lombok.SneakyThrows; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.AuthAction; +import org.apache.pulsar.common.policies.data.RetentionPolicies; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.security.MockedPulsarStandalone; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +public final class TopicPoliciesAuthZTest extends MockedPulsarStandalone { + + private PulsarAdmin superUserAdmin; + + private PulsarAdmin tenantManagerAdmin; + + private static final String TENANT_ADMIN_SUBJECT = UUID.randomUUID().toString(); + private static final String TENANT_ADMIN_TOKEN = Jwts.builder() + .claim("sub", TENANT_ADMIN_SUBJECT).signWith(SECRET_KEY).compact(); + + @SneakyThrows + @BeforeClass + public void before() { + configureTokenAuthentication(); + configureDefaultAuthorization(); + start(); + this.superUserAdmin =PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(SUPER_USER_TOKEN)) + .build(); + final TenantInfo tenantInfo = superUserAdmin.tenants().getTenantInfo("public"); + tenantInfo.getAdminRoles().add(TENANT_ADMIN_SUBJECT); + superUserAdmin.tenants().updateTenant("public", tenantInfo); + this.tenantManagerAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(TENANT_ADMIN_TOKEN)) + .build(); + } + + + @SneakyThrows + @AfterClass + public void after() { + close(); + } + + + @SneakyThrows + @Test + public void testRetention() { + final String random = UUID.randomUUID().toString(); + final String topic = "persistent://public/default/" + random; + final String subject = UUID.randomUUID().toString(); + final String token = Jwts.builder() + .claim("sub", subject).signWith(SECRET_KEY).compact(); + superUserAdmin.topics().createNonPartitionedTopic(topic); + + @Cleanup + final PulsarAdmin subAdmin = PulsarAdmin.builder() + .serviceHttpUrl(getPulsarService().getWebServiceAddress()) + .authentication(new AuthenticationToken(token)) + .build(); + final RetentionPolicies definedRetentionPolicy = new RetentionPolicies(1, 1); + // test superuser + superUserAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + + // because the topic policies is eventual consistency, we should wait here + await().untilAsserted(() -> { + final RetentionPolicies receivedRetentionPolicy = superUserAdmin.topicPolicies().getRetention(topic); + Assert.assertEquals(receivedRetentionPolicy, definedRetentionPolicy); + }); + superUserAdmin.topicPolicies().removeRetention(topic); + + await().untilAsserted(() -> { + final RetentionPolicies retention = superUserAdmin.topicPolicies().getRetention(topic); + Assert.assertNull(retention); + }); + + // test tenant manager + + tenantManagerAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + await().untilAsserted(() -> { + final RetentionPolicies receivedRetentionPolicy = tenantManagerAdmin.topicPolicies().getRetention(topic); + Assert.assertEquals(receivedRetentionPolicy, definedRetentionPolicy); + }); + tenantManagerAdmin.topicPolicies().removeRetention(topic); + await().untilAsserted(() -> { + final RetentionPolicies retention = tenantManagerAdmin.topicPolicies().getRetention(topic); + Assert.assertNull(retention); + }); + + // test nobody + + try { + subAdmin.topicPolicies().getRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + // test sub user with permissions + for (AuthAction action : AuthAction.values()) { + superUserAdmin.namespaces().grantPermissionOnNamespace("public/default", + subject, Set.of(action)); + try { + subAdmin.topicPolicies().getRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + + subAdmin.topicPolicies().setRetention(topic, definedRetentionPolicy); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + + try { + subAdmin.topicPolicies().removeRetention(topic); + Assert.fail("unexpected behaviour"); + } catch (PulsarAdminException ex) { + Assert.assertTrue(ex instanceof PulsarAdminException.NotAuthorizedException); + } + superUserAdmin.namespaces().revokePermissionsOnNamespace("public/default", subject); + } + } + +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java b/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java new file mode 100644 index 0000000000000..20dd2e1066a87 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.security; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.SignatureAlgorithm; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import javax.crypto.SecretKey; +import lombok.Getter; +import lombok.SneakyThrows; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.authentication.AuthenticationProviderToken; +import org.apache.pulsar.broker.authentication.utils.AuthTokenUtils; +import org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider; +import org.apache.pulsar.broker.testcontext.PulsarTestContext; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.impl.auth.AuthenticationToken; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfo; +import org.apache.pulsar.common.util.ObjectMapperFactory; + + +public abstract class MockedPulsarStandalone implements AutoCloseable { + + @Getter + private final ServiceConfiguration serviceConfiguration = new ServiceConfiguration(); + private PulsarTestContext pulsarTestContext; + + @Getter + private PulsarService pulsarService; + private PulsarAdmin serviceInternalAdmin; + + + { + serviceConfiguration.setClusterName(TEST_CLUSTER_NAME); + serviceConfiguration.setBrokerShutdownTimeoutMs(0L); + serviceConfiguration.setBrokerServicePort(Optional.of(0)); + serviceConfiguration.setBrokerServicePortTls(Optional.of(0)); + serviceConfiguration.setAdvertisedAddress("localhost"); + serviceConfiguration.setWebServicePort(Optional.of(0)); + serviceConfiguration.setWebServicePortTls(Optional.of(0)); + serviceConfiguration.setNumExecutorThreadPoolSize(5); + serviceConfiguration.setExposeBundlesMetricsInPrometheus(true); + } + + + protected static final SecretKey SECRET_KEY = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256); + + private static final String BROKER_INTERNAL_CLIENT_SUBJECT = "broker_internal"; + private static final String BROKER_INTERNAL_CLIENT_TOKEN = Jwts.builder() + .claim("sub", BROKER_INTERNAL_CLIENT_SUBJECT).signWith(SECRET_KEY).compact(); + protected static final String SUPER_USER_SUBJECT = "super-user"; + protected static final String SUPER_USER_TOKEN = Jwts.builder() + .claim("sub", SUPER_USER_SUBJECT).signWith(SECRET_KEY).compact(); + protected static final String NOBODY_SUBJECT = "nobody"; + protected static final String NOBODY_TOKEN = Jwts.builder() + .claim("sub", NOBODY_SUBJECT).signWith(SECRET_KEY).compact(); + + + @SneakyThrows + protected void configureTokenAuthentication() { + serviceConfiguration.setAuthenticationEnabled(true); + serviceConfiguration.setAuthenticationProviders(Set.of(AuthenticationProviderToken.class.getName())); + // internal client + serviceConfiguration.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName()); + final Map brokerClientAuthParams = new HashMap<>(); + brokerClientAuthParams.put("token", BROKER_INTERNAL_CLIENT_TOKEN); + final String brokerClientAuthParamStr = MAPPER.writeValueAsString(brokerClientAuthParams); + serviceConfiguration.setBrokerClientAuthenticationParameters(brokerClientAuthParamStr); + + Properties properties = serviceConfiguration.getProperties(); + if (properties == null) { + properties = new Properties(); + serviceConfiguration.setProperties(properties); + } + properties.put("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(SECRET_KEY)); + + } + + + + protected void configureDefaultAuthorization() { + serviceConfiguration.setAuthorizationEnabled(true); + serviceConfiguration.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); + serviceConfiguration.setSuperUserRoles(Set.of(SUPER_USER_SUBJECT, BROKER_INTERNAL_CLIENT_SUBJECT)); + } + + + @SneakyThrows + protected void start() { + this.pulsarTestContext = PulsarTestContext.builder() + .spyByDefault() + .config(serviceConfiguration) + .withMockZookeeper(false) + .build(); + this.pulsarService = pulsarTestContext.getPulsarService(); + this.serviceInternalAdmin = pulsarService.getAdminClient(); + setupDefaultTenantAndNamespace(); + } + + private void setupDefaultTenantAndNamespace() throws Exception { + if (!serviceInternalAdmin.clusters().getClusters().contains(TEST_CLUSTER_NAME)) { + serviceInternalAdmin.clusters().createCluster(TEST_CLUSTER_NAME, + ClusterData.builder().serviceUrl(pulsarService.getWebServiceAddress()).build()); + } + if (!serviceInternalAdmin.tenants().getTenants().contains(DEFAULT_TENANT)) { + serviceInternalAdmin.tenants().createTenant(DEFAULT_TENANT, TenantInfo.builder().allowedClusters( + Sets.newHashSet(TEST_CLUSTER_NAME)).build()); + } + if (!serviceInternalAdmin.namespaces().getNamespaces(DEFAULT_TENANT).contains(DEFAULT_NAMESPACE)) { + serviceInternalAdmin.namespaces().createNamespace(DEFAULT_NAMESPACE); + } + } + + + @Override + public void close() throws Exception { + if (pulsarTestContext != null) { + pulsarTestContext.close(); + } + } + + // Utils + protected static final ObjectMapper mapper = new ObjectMapper(); + + // Static name + private static final String DEFAULT_TENANT = "public"; + private static final String DEFAULT_NAMESPACE = "public/default"; + private static final String TEST_CLUSTER_NAME = "test-standalone"; + + private static final ObjectMapper MAPPER = ObjectMapperFactory.getMapper().getObjectMapper(); +} From 4252cf77174d1ce6eb958f602e25cf874c16c78a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Mon, 4 Mar 2024 18:12:08 +0200 Subject: [PATCH 493/494] [improve][fn] Add configuration for connector & functions package url sources (#22184) (cherry picked from commit 207335a449f2bc9cdf6782c67f93f8c2fb267271) --- conf/functions_worker.yml | 8 ++ .../worker/PulsarFunctionE2ESecurityTest.java | 6 ++ .../worker/PulsarFunctionPublishTest.java | 4 + .../worker/PulsarFunctionTlsTest.java | 9 +- .../pulsar/io/AbstractPulsarE2ETest.java | 6 ++ .../pulsar/functions/worker/WorkerConfig.java | 24 +++++ .../functions/worker/FunctionActioner.java | 16 ++- .../worker/FunctionRuntimeManager.java | 3 +- .../functions/worker/PackageUrlValidator.java | 101 ++++++++++++++++++ .../functions/worker/PulsarWorkerService.java | 4 +- .../worker/rest/api/ComponentImpl.java | 32 ++++-- .../worker/rest/api/FunctionsImpl.java | 3 +- .../functions/worker/rest/api/SinksImpl.java | 9 +- .../worker/rest/api/SourcesImpl.java | 3 +- .../worker/FunctionActionerTest.java | 17 ++- .../worker/FunctionRuntimeManagerTest.java | 2 +- .../v3/AbstractFunctionApiResourceTest.java | 7 ++ .../api/v3/AbstractFunctionsResourceTest.java | 18 +++- .../rest/api/v3/SinkApiV3ResourceTest.java | 56 +++++----- .../rest/api/v3/SourceApiV3ResourceTest.java | 9 ++ 20 files changed, 282 insertions(+), 55 deletions(-) create mode 100644 pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PackageUrlValidator.java diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 05c875adc4c64..460becd9a94a3 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -408,7 +408,15 @@ saslJaasServerRoleTokenSignerSecretPath: ######################## connectorsDirectory: ./connectors +# Whether to enable referencing connectors directory files by file url in connector (sink/source) creation +enableReferencingConnectorDirectoryFiles: true +# Regex patterns for enabling creation of connectors by referencing packages in matching http/https urls +additionalEnabledConnectorUrlPatterns: [] functionsDirectory: ./functions +# Whether to enable referencing functions directory files by file url in functions creation +enableReferencingFunctionsDirectoryFiles: true +# Regex patterns for enabling creation of functions by referencing packages in matching http/https urls +additionalEnabledFunctionsUrlPatterns: [] # Enables extended validation for connector config with fine-grain annotation based validation # during submission. Classloading with either enableClassloadingOfExternalFiles or diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java index 107aedd076691..cbf2f28b0b50b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionE2ESecurityTest.java @@ -34,6 +34,7 @@ import java.net.URL; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; @@ -267,6 +268,11 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthorizationEnabled(config.isAuthorizationEnabled()); workerConfig.setAuthorizationProvider(config.getAuthorizationProvider()); + List urlPatterns = + List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*", "http://127\\.0\\.0\\.1:.*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java index 7bcf1dec871e0..72e6a3766aeee 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionPublishTest.java @@ -268,6 +268,10 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthenticationEnabled(true); workerConfig.setAuthorizationEnabled(true); + List urlPatterns = List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); return workerService; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index 9882b15450e40..3508cf0bfc7e6 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -20,8 +20,8 @@ import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -153,6 +154,12 @@ void setup() throws Exception { workerConfig.setUseTls(true); workerConfig.setTlsEnableHostnameVerification(true); workerConfig.setTlsAllowInsecureConnection(false); + File packagePath = new File( + PulsarSink.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParentFile(); + List urlPatterns = + List.of(packagePath.toURI() + ".*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); fnWorkerServices[i] = WorkerServiceLoader.load(workerConfig); configurations[i] = config; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java index f968315a7124c..3c0dd0822b7dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/AbstractPulsarE2ETest.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -306,6 +307,11 @@ private PulsarWorkerService createPulsarFunctionWorker(ServiceConfiguration conf workerConfig.setAuthenticationEnabled(true); workerConfig.setAuthorizationEnabled(true); + List urlPatterns = + List.of(getPulsarApiExamplesJar().getParentFile().toURI() + ".*", "http://127\\.0\\.0\\.1:.*"); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(urlPatterns); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(urlPatterns); + PulsarWorkerService workerService = new PulsarWorkerService(); return workerService; } diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java index 8dfd91e0b20cf..f88df1bd24c3f 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java @@ -264,6 +264,18 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { doc = "The directory where nar packages are extractors" ) private String narExtractionDirectory = NarClassLoader.DEFAULT_NAR_EXTRACTION_DIR; + @FieldContext( + category = CATEGORY_CONNECTORS, + doc = "Whether to enable referencing connectors directory files by file url in connector (sink/source) " + + "creation. Default is true." + ) + private Boolean enableReferencingConnectorDirectoryFiles = true; + @FieldContext( + category = CATEGORY_FUNCTIONS, + doc = "Regex patterns for enabling creation of connectors by referencing packages in matching http/https " + + "urls." + ) + private List additionalEnabledConnectorUrlPatterns = new ArrayList<>(); @FieldContext( category = CATEGORY_CONNECTORS, doc = "Enables extended validation for connector config with fine-grain annotation based validation " @@ -282,6 +294,18 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { doc = "The path to the location to locate builtin functions" ) private String functionsDirectory = "./functions"; + @FieldContext( + category = CATEGORY_FUNCTIONS, + doc = "Whether to enable referencing functions directory files by file url in functions creation. " + + "Default is true." + ) + private Boolean enableReferencingFunctionsDirectoryFiles = true; + @FieldContext( + category = CATEGORY_FUNCTIONS, + doc = "Regex patterns for enabling creation of functions by referencing packages in matching http/https " + + "urls." + ) + private List additionalEnabledFunctionsUrlPatterns = new ArrayList<>(); @FieldContext( category = CATEGORY_FUNC_METADATA_MNG, doc = "The Pulsar topic used for storing function metadata" diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index 6c794317cf10d..217789a1077b1 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -83,18 +83,21 @@ public class FunctionActioner { private final ConnectorsManager connectorsManager; private final FunctionsManager functionsManager; private final PulsarAdmin pulsarAdmin; + private final PackageUrlValidator packageUrlValidator; public FunctionActioner(WorkerConfig workerConfig, RuntimeFactory runtimeFactory, Namespace dlogNamespace, ConnectorsManager connectorsManager, - FunctionsManager functionsManager, PulsarAdmin pulsarAdmin) { + FunctionsManager functionsManager, PulsarAdmin pulsarAdmin, + PackageUrlValidator packageUrlValidator) { this.workerConfig = workerConfig; this.runtimeFactory = runtimeFactory; this.dlogNamespace = dlogNamespace; this.connectorsManager = connectorsManager; this.functionsManager = functionsManager; this.pulsarAdmin = pulsarAdmin; + this.packageUrlValidator = packageUrlValidator; } @@ -152,6 +155,9 @@ private String getPackageFile(FunctionMetaData functionMetaData, FunctionDetails boolean isPkgUrlProvided = isFunctionPackageUrlSupported(packagePath); String packageFile; if (isPkgUrlProvided && packagePath.startsWith(FILE)) { + if (!packageUrlValidator.isValidPackageUrl(componentType, packagePath)) { + throw new IllegalArgumentException("Package URL " + packagePath + " is not valid"); + } URL url = new URL(packagePath); File pkgFile = new File(url.toURI()); packageFile = pkgFile.getAbsolutePath(); @@ -168,7 +174,7 @@ private String getPackageFile(FunctionMetaData functionMetaData, FunctionDetails pkgDir, new File(getDownloadFileName(functionMetaData.getFunctionDetails(), pkgLocation)).getName()); - downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId, pkgLocation); + downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId, pkgLocation, componentType); packageFile = pkgFile.getAbsolutePath(); } return packageFile; @@ -226,7 +232,8 @@ InstanceConfig createInstanceConfig(FunctionDetails functionDetails, Function.Fu } private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaData functionMetaData, - int instanceId, Function.PackageLocationMetaData pkgLocation) + int instanceId, Function.PackageLocationMetaData pkgLocation, + FunctionDetails.ComponentType componentType) throws IOException, PulsarAdminException { FunctionDetails details = functionMetaData.getFunctionDetails(); @@ -251,6 +258,9 @@ private void downloadFile(File pkgFile, boolean isPkgUrlProvided, FunctionMetaDa downloadFromHttp ? pkgLocationPath : pkgLocation); if (downloadFromHttp) { + if (!packageUrlValidator.isValidPackageUrl(componentType, pkgLocationPath)) { + throw new IllegalArgumentException("Package URL " + pkgLocationPath + " is not valid"); + } FunctionCommon.downloadFromHttpUrl(pkgLocationPath, tempPkgFile); } else if (downloadFromPackageManagementService) { getPulsarAdmin().packages().download(pkgLocationPath, tempPkgFile.getPath()); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index 8e6725e93af91..b6e2bbb1ca0f8 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -220,7 +220,8 @@ public FunctionRuntimeManager(WorkerConfig workerConfig, PulsarWorkerService wor functionAuthProvider, runtimeCustomizer); this.functionActioner = new FunctionActioner(this.workerConfig, runtimeFactory, - dlogNamespace, connectorsManager, functionsManager, workerService.getBrokerAdmin()); + dlogNamespace, connectorsManager, functionsManager, workerService.getBrokerAdmin(), + workerService.getPackageUrlValidator()); this.membershipManager = membershipManager; this.functionMetaDataManager = functionMetaDataManager; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PackageUrlValidator.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PackageUrlValidator.java new file mode 100644 index 0000000000000..2a8fe8dddb153 --- /dev/null +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PackageUrlValidator.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker; + +import java.net.URI; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.pulsar.functions.proto.Function; + +/** + * Validates package URLs for functions and connectors. + * Validates that the package URL is either a file in the connectors or functions directory + * when referencing connector or function files is enabled, or matches one of the additional url patterns. + */ +public class PackageUrlValidator { + private final Path connectionsDirectory; + private final Path functionsDirectory; + private final List additionalConnectionsPatterns; + private final List additionalFunctionsPatterns; + + public PackageUrlValidator(WorkerConfig workerConfig) { + this.connectionsDirectory = resolveDirectory(workerConfig.getEnableReferencingConnectorDirectoryFiles(), + workerConfig.getConnectorsDirectory()); + this.functionsDirectory = resolveDirectory(workerConfig.getEnableReferencingFunctionsDirectoryFiles(), + workerConfig.getFunctionsDirectory()); + this.additionalConnectionsPatterns = + compilePatterns(workerConfig.getAdditionalEnabledConnectorUrlPatterns()); + this.additionalFunctionsPatterns = + compilePatterns(workerConfig.getAdditionalEnabledFunctionsUrlPatterns()); + } + + private static Path resolveDirectory(Boolean enabled, String directory) { + return enabled != null && enabled + ? Path.of(directory).normalize().toAbsolutePath() : null; + } + + private static List compilePatterns(List additionalPatterns) { + return additionalPatterns != null ? additionalPatterns.stream().map(Pattern::compile).collect( + Collectors.toList()) : Collections.emptyList(); + } + + boolean isValidFunctionsPackageUrl(URI functionPkgUrl) { + return doesMatch(functionPkgUrl, functionsDirectory, additionalFunctionsPatterns); + } + + boolean isValidConnectionsPackageUrl(URI functionPkgUrl) { + return doesMatch(functionPkgUrl, connectionsDirectory, additionalConnectionsPatterns); + } + + private boolean doesMatch(URI functionPkgUrl, Path directory, List patterns) { + if (directory != null && "file".equals(functionPkgUrl.getScheme())) { + Path filePath = Path.of(functionPkgUrl.getPath()).normalize().toAbsolutePath(); + if (filePath.startsWith(directory)) { + return true; + } + } + String functionPkgUrlString = functionPkgUrl.normalize().toString(); + for (Pattern pattern : patterns) { + if (pattern.matcher(functionPkgUrlString).matches()) { + return true; + } + } + return false; + } + + public boolean isValidPackageUrl(Function.FunctionDetails.ComponentType componentType, String functionPkgUrl) { + URI uri = URI.create(functionPkgUrl); + if (componentType == null) { + // if component type is not specified, we need to check both functions and connections + return isValidFunctionsPackageUrl(uri) || isValidConnectionsPackageUrl(uri); + } + switch (componentType) { + case FUNCTION: + return isValidFunctionsPackageUrl(uri); + case SINK: + case SOURCE: + return isValidConnectionsPackageUrl(uri); + default: + throw new IllegalArgumentException("Unknown component type: " + componentType); + } + } +} diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java index f9f2738828be7..84b943e5671ac 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/PulsarWorkerService.java @@ -119,7 +119,8 @@ public interface PulsarClientCreator { private Sinks sinks; private Sources sources; private Workers workers; - + @Getter + private PackageUrlValidator packageUrlValidator; private final PulsarClientCreator clientCreator; public PulsarWorkerService() { @@ -196,6 +197,7 @@ public void init(WorkerConfig workerConfig, this.sinks = new SinksImpl(() -> PulsarWorkerService.this); this.sources = new SourcesImpl(() -> PulsarWorkerService.this); this.workers = new WorkerImpl(() -> PulsarWorkerService.this); + this.packageUrlValidator = new PackageUrlValidator(workerConfig); } @Override diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 5e105f7057e33..6d07e5870917a 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -1376,11 +1376,17 @@ private StreamingOutput getStreamingOutput(String pkgPath) { private StreamingOutput getStreamingOutput(String pkgPath, FunctionDetails.ComponentType componentType) { return output -> { if (pkgPath.startsWith(Utils.HTTP)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, pkgPath)) { + throw new IllegalArgumentException("Invalid package url: " + pkgPath); + } URL url = URI.create(pkgPath).toURL(); try (InputStream inputStream = url.openStream()) { IOUtils.copy(inputStream, output); } } else if (pkgPath.startsWith(Utils.FILE)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, pkgPath)) { + throw new IllegalArgumentException("Invalid package url: " + pkgPath); + } URI url = URI.create(pkgPath); File file = new File(url.getPath()); Files.copy(file.toPath(), output); @@ -1804,12 +1810,17 @@ static File downloadPackageFile(PulsarWorkerService worker, String packageName) return file; } - protected File getPackageFile(String functionPkgUrl, String existingPackagePath, InputStream uploadedInputStream) + protected File getPackageFile(FunctionDetails.ComponentType componentType, String functionPkgUrl, + String existingPackagePath, InputStream uploadedInputStream) throws IOException, PulsarAdminException { File componentPackageFile = null; if (isNotBlank(functionPkgUrl)) { - componentPackageFile = getPackageFile(functionPkgUrl); + componentPackageFile = getPackageFile(componentType, functionPkgUrl); } else if (existingPackagePath.startsWith(Utils.FILE) || existingPackagePath.startsWith(Utils.HTTP)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, functionPkgUrl)) { + throw new IllegalArgumentException("Function Package url is not valid." + + "supported url (http/https/file)"); + } try { componentPackageFile = FunctionCommon.extractFileFromPkgURL(existingPackagePath); } catch (Exception e) { @@ -1818,7 +1829,7 @@ protected File getPackageFile(String functionPkgUrl, String existingPackagePath, ComponentTypeUtils.toString(componentType), functionPkgUrl)); } } else if (Utils.hasPackageTypePrefix(existingPackagePath)) { - componentPackageFile = getPackageFile(existingPackagePath); + componentPackageFile = getPackageFile(componentType, existingPackagePath); } else if (uploadedInputStream != null) { componentPackageFile = WorkerUtils.dumpToTmpFile(uploadedInputStream); } else if (!existingPackagePath.startsWith(Utils.BUILTIN)) { @@ -1836,15 +1847,16 @@ protected File getPackageFile(String functionPkgUrl, String existingPackagePath, return componentPackageFile; } - protected File downloadPackageFile(String packageName) throws IOException, PulsarAdminException { - return downloadPackageFile(worker(), packageName); - } - - protected File getPackageFile(String functionPkgUrl) throws IOException, PulsarAdminException { + protected File getPackageFile(FunctionDetails.ComponentType componentType, String functionPkgUrl) + throws IOException, PulsarAdminException { if (Utils.hasPackageTypePrefix(functionPkgUrl)) { - return downloadPackageFile(functionPkgUrl); + if (!worker().getWorkerConfig().isFunctionsWorkerEnablePackageManagement()) { + throw new IllegalStateException("Function Package management service is disabled. " + + "Please enable it to use " + functionPkgUrl); + } + return downloadPackageFile(worker(), functionPkgUrl); } else { - if (!Utils.isFunctionPackageUrlSupported(functionPkgUrl)) { + if (!worker().getPackageUrlValidator().isValidPackageUrl(componentType, functionPkgUrl)) { throw new IllegalArgumentException("Function Package url is not valid." + "supported url (http/https/file)"); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 6b81d2c4918a6..a075d3e18a0b3 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -144,7 +144,7 @@ public void registerFunction(final String tenant, // validate parameters try { if (isNotBlank(functionPkgUrl)) { - componentPackageFile = getPackageFile(functionPkgUrl); + componentPackageFile = getPackageFile(componentType, functionPkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, functionName, functionConfig, componentPackageFile); } else { @@ -305,6 +305,7 @@ public void updateFunction(final String tenant, // validate parameters try { componentPackageFile = getPackageFile( + componentType, functionPkgUrl, existingComponent.getPackageLocation().getPackagePath(), uploadedInputStream); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index 51d1333a79c36..6b8b41e5a8e5b 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -143,7 +143,7 @@ public void registerSink(final String tenant, // validate parameters try { if (isNotBlank(sinkPkgUrl)) { - componentPackageFile = getPackageFile(sinkPkgUrl); + componentPackageFile = getPackageFile(componentType, sinkPkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, sinkName, sinkConfig, componentPackageFile); } else { @@ -310,6 +310,7 @@ public void updateSink(final String tenant, // validate parameters try { componentPackageFile = getPackageFile( + componentType, sinkPkgUrl, existingComponent.getPackageLocation().getPackagePath(), uploadedInputStream); @@ -421,7 +422,8 @@ private void setTransformFunctionPackageLocation(Function.FunctionMetaData.Build try { String builtin = functionDetails.getBuiltin(); if (isBlank(builtin)) { - functionPackageFile = getPackageFile(transformFunction); + functionPackageFile = + getPackageFile(Function.FunctionDetails.ComponentType.FUNCTION, transformFunction); } Function.PackageLocationMetaData.Builder functionPackageLocation = getFunctionPackageLocation(functionMetaDataBuilder.build(), @@ -744,7 +746,8 @@ private Function.FunctionDetails validateUpdateRequestParams(final String tenant transformFunctionPackage = getBuiltinFunctionPackage(sinkConfig.getTransformFunction()); if (transformFunctionPackage == null) { - File functionPackageFile = getPackageFile(sinkConfig.getTransformFunction()); + File functionPackageFile = getPackageFile(Function.FunctionDetails.ComponentType.FUNCTION, + sinkConfig.getTransformFunction()); transformFunctionPackage = new FunctionFilePackage(functionPackageFile, workerConfig.getNarExtractionDirectory(), workerConfig.getEnableClassloadingOfExternalFiles(), ConnectorDefinition.class); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java index dea69698dd28d..5191306146951 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SourcesImpl.java @@ -143,7 +143,7 @@ public void registerSource(final String tenant, // validate parameters try { if (isPkgUrlProvided) { - componentPackageFile = getPackageFile(sourcePkgUrl); + componentPackageFile = getPackageFile(componentType, sourcePkgUrl); functionDetails = validateUpdateRequestParams(tenant, namespace, sourceName, sourceConfig, componentPackageFile); } else { @@ -304,6 +304,7 @@ public void updateSource(final String tenant, // validate parameters try { componentPackageFile = getPackageFile( + componentType, sourcePkgUrl, existingComponent.getPackageLocation().getPackagePath(), uploadedInputStream); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java index 4e4c3d2f234aa..ac5ca617ea43b 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionActionerTest.java @@ -30,6 +30,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.fail; +import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.distributedlog.api.namespace.Namespace; @@ -77,7 +78,8 @@ public void testStartFunctionWithDLNamespace() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, - new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class)); + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class), + mock(PackageUrlValidator.class)); Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder() .setFunctionDetails(Function.FunctionDetails.newBuilder().setTenant("test-tenant") .setNamespace("test-namespace").setName("func-1")) @@ -109,6 +111,8 @@ public void testStartFunctionWithPkgUrl() throws Exception { workerConfig.setPulsarServiceUrl("pulsar://localhost:6650"); workerConfig.setStateStorageServiceUrl("foo"); workerConfig.setFunctionAssignmentTopicName("assignments"); + workerConfig.setAdditionalEnabledFunctionsUrlPatterns(List.of("file:///user/.*", "http://invalid/.*")); + workerConfig.setAdditionalEnabledConnectorUrlPatterns(List.of("file:///user/.*", "http://invalid/.*")); String downloadDir = this.getClass().getProtectionDomain().getCodeSource().getLocation().getPath(); workerConfig.setDownloadDirectory(downloadDir); @@ -122,11 +126,12 @@ public void testStartFunctionWithPkgUrl() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, - new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class)); + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class), + new PackageUrlValidator(workerConfig)); // (1) test with file url. functionActioner should be able to consider file-url and it should be able to call // RuntimeSpawner - String pkgPathLocation = FILE + ":/user/my-file.jar"; + String pkgPathLocation = FILE + ":///user/my-file.jar"; startFunction(actioner, pkgPathLocation, pkgPathLocation); verify(runtime, times(1)).start(); @@ -194,7 +199,8 @@ public void testFunctionAuthDisabled() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, - new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class)); + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), mock(PulsarAdmin.class), + mock(PackageUrlValidator.class)); String pkgPathLocation = "http://invalid/my-file.jar"; @@ -257,7 +263,8 @@ public void testStartFunctionWithPackageUrl() throws Exception { @SuppressWarnings("resource") FunctionActioner actioner = new FunctionActioner(workerConfig, factory, dlogNamespace, - new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), pulsarAdmin); + new ConnectorsManager(workerConfig), new FunctionsManager(workerConfig), pulsarAdmin, + mock(PackageUrlValidator.class)); // (1) test with file url. functionActioner should be able to consider file-url and it should be able to call // RuntimeSpawner diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java index bc56b1766d39d..7b57a1c896539 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java @@ -720,7 +720,7 @@ public void testExternallyManagedRuntimeUpdate() throws Exception { FunctionActioner functionActioner = spy(new FunctionActioner( workerConfig, - kubernetesRuntimeFactory, null, null, null, null)); + kubernetesRuntimeFactory, null, null, null, null, workerService.getPackageUrlValidator())); try (final MockedStatic runtimeFactoryMockedStatic = Mockito .mockStatic(RuntimeFactory.class);) { diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java index 5845ff3afd9ac..388331ce6f241 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionApiResourceTest.java @@ -29,6 +29,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -59,6 +60,12 @@ import org.testng.annotations.Test; public abstract class AbstractFunctionApiResourceTest extends AbstractFunctionsResourceTest { + @Override + protected void customizeWorkerConfig(WorkerConfig workerConfig, Method method) { + if (method.getName().contains("Upload")) { + workerConfig.setFunctionsWorkerEnablePackageManagement(false); + } + } @Test public void testListFunctionsSuccess() { diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java index 4cc4ed0b09819..51ca4c83f9e02 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/AbstractFunctionsResourceTest.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -63,6 +64,7 @@ import org.apache.pulsar.functions.worker.FunctionRuntimeManager; import org.apache.pulsar.functions.worker.FunctionsManager; import org.apache.pulsar.functions.worker.LeaderService; +import org.apache.pulsar.functions.worker.PackageUrlValidator; import org.apache.pulsar.functions.worker.PulsarWorkerService; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; @@ -141,7 +143,7 @@ public static File getPulsarApiExamplesNar() { } @BeforeMethod - public final void setup() throws Exception { + public final void setup(Method method) throws Exception { this.mockedManager = mock(FunctionMetaDataManager.class); this.mockedFunctionRunTimeManager = mock(FunctionRuntimeManager.class); this.mockedRuntimeFactory = mock(RuntimeFactory.class); @@ -181,21 +183,33 @@ public final void setup() throws Exception { }).when(mockedPackages).download(any(), any()); // worker config + List urlPatterns = + List.of("http://localhost.*", "file:.*", "https://repo1.maven.org/maven2/org/apache/pulsar/.*"); WorkerConfig workerConfig = new WorkerConfig() .setWorkerId("test") .setWorkerPort(8080) .setFunctionMetadataTopicName("pulsar/functions") .setNumFunctionPackageReplicas(3) - .setPulsarServiceUrl("pulsar://localhost:6650/"); + .setPulsarServiceUrl("pulsar://localhost:6650/") + .setAdditionalEnabledFunctionsUrlPatterns(urlPatterns) + .setAdditionalEnabledConnectorUrlPatterns(urlPatterns) + .setFunctionsWorkerEnablePackageManagement(true); + customizeWorkerConfig(workerConfig, method); tempDirectory = PulsarFunctionTestTemporaryDirectory.create(getClass().getSimpleName()); tempDirectory.useTemporaryDirectoriesForWorkerConfig(workerConfig); when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); when(mockedWorkerService.getFunctionsManager()).thenReturn(functionsManager); when(mockedWorkerService.getConnectorsManager()).thenReturn(connectorsManager); + PackageUrlValidator packageUrlValidator = new PackageUrlValidator(workerConfig); + when(mockedWorkerService.getPackageUrlValidator()).thenReturn(packageUrlValidator); doSetup(); } + protected void customizeWorkerConfig(WorkerConfig workerConfig, Method method) { + + } + protected File getDefaultNarFile() { return getPulsarIOTwitterNar(); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index b9833380d7087..c6c6303e48007 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -36,6 +36,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -58,6 +59,7 @@ import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.Function.FunctionMetaData; import org.apache.pulsar.functions.utils.SinkConfigUtils; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.rest.api.SinksImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -80,6 +82,12 @@ protected void doSetup() { this.resource = spy(new SinksImpl(() -> mockedWorkerService)); } + @Override + protected void customizeWorkerConfig(WorkerConfig workerConfig, Method method) { + if (method.getName().contains("Upload") || method.getName().contains("BKPackage")) { + workerConfig.setFunctionsWorkerEnablePackageManagement(false); + } + } @Override protected Function.FunctionDetails.ComponentType getComponentType() { return Function.FunctionDetails.ComponentType.SINK; @@ -1486,17 +1494,15 @@ public void testRegisterSinkSuccessK8sNoUpload() throws Exception { SinkConfig sinkConfig = createDefaultSinkConfig(); sinkConfig.setArchive("builtin://cassandra"); - try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { - resource.registerSink( - tenant, - namespace, - sink, - inputStream, - mockedFormData, - null, - sinkConfig, - null); - } + resource.registerSink( + tenant, + namespace, + sink, + null, + mockedFormData, + null, + sinkConfig, + null); } /* @@ -1526,21 +1532,19 @@ public void testRegisterSinkSuccessK8sWithUpload() throws Exception { SinkConfig sinkConfig = createDefaultSinkConfig(); sinkConfig.setArchive("builtin://cassandra"); - try (FileInputStream inputStream = new FileInputStream(getPulsarIOCassandraNar())) { - try { - resource.registerSink( - tenant, - namespace, - sink, - inputStream, - mockedFormData, - null, - sinkConfig, - null); - Assert.fail(); - } catch (RuntimeException e) { - Assert.assertEquals(e.getMessage(), injectedErrMsg); - } + try { + resource.registerSink( + tenant, + namespace, + sink, + null, + mockedFormData, + null, + sinkConfig, + null); + Assert.fail(); + } catch (RuntimeException e) { + Assert.assertEquals(e.getMessage(), injectedErrMsg); } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java index c7e69484d3019..f02acbd3663bf 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SourceApiV3ResourceTest.java @@ -34,6 +34,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.util.LinkedList; import java.util.List; import javax.ws.rs.core.Response; @@ -57,6 +58,7 @@ import org.apache.pulsar.functions.source.TopicSchema; import org.apache.pulsar.functions.utils.SourceConfigUtils; import org.apache.pulsar.functions.utils.io.ConnectorUtils; +import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerUtils; import org.apache.pulsar.functions.worker.rest.api.SourcesImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -80,6 +82,13 @@ protected void doSetup() { this.resource = spy(new SourcesImpl(() -> mockedWorkerService)); } + @Override + protected void customizeWorkerConfig(WorkerConfig workerConfig, Method method) { + if (method.getName().endsWith("UploadFailure") || method.getName().contains("BKPackage")) { + workerConfig.setFunctionsWorkerEnablePackageManagement(false); + } + } + @Override protected FunctionDetails.ComponentType getComponentType() { return FunctionDetails.ComponentType.SOURCE; From f93afef49e08fd92df837cc82fa3f419c37fd523 Mon Sep 17 00:00:00 2001 From: StreamNative Bot Date: Mon, 4 Mar 2024 21:08:13 +0000 Subject: [PATCH 494/494] Release 3.0.0-SNAPSHOT --- .idea/vcs.xml | 28 +- .mvn/extensions.xml | 5 +- .mvn/gradle-enterprise.xml | 6 +- bouncy-castle/bc/pom.xml | 16 +- bouncy-castle/bcfips-include-test/pom.xml | 14 +- bouncy-castle/bcfips/pom.xml | 14 +- bouncy-castle/pom.xml | 12 +- buildtools/pom.xml | 79 +- buildtools/src/main/resources/log4j2.xml | 32 +- .../src/main/resources/pulsar/checkstyle.xml | 580 +++-- .../main/resources/pulsar/suppressions.xml | 121 +- conf/filesystem_offload_core_site.xml | 52 +- conf/functions_log4j2.xml | 217 +- distribution/io/pom.xml | 38 +- distribution/io/src/assemble/io.xml | 155 +- distribution/offloaders/pom.xml | 36 +- .../offloaders/src/assemble/offloaders.xml | 7 +- distribution/pom.xml | 38 +- distribution/server/pom.xml | 65 +- distribution/server/src/assemble/bin.xml | 12 +- distribution/shell/pom.xml | 40 +- distribution/shell/src/assemble/shell.xml | 7 +- docker/pom.xml | 33 +- docker/pulsar-all/pom.xml | 12 +- docker/pulsar/pom.xml | 13 +- jclouds-shaded/pom.xml | 33 +- managed-ledger/pom.xml | 36 +- pom.xml | 549 ++--- pulsar-broker-auth-athenz/pom.xml | 21 +- .../src/test/resources/findbugsExclude.xml | 3 +- pulsar-broker-auth-oidc/pom.xml | 27 +- pulsar-broker-auth-sasl/pom.xml | 23 +- pulsar-broker-common/pom.xml | 23 +- pulsar-broker/pom.xml | 97 +- pulsar-client-1x-base/pom.xml | 14 +- .../pulsar-client-1x/pom.xml | 17 +- .../src/main/resources/findbugsExclude.xml | 1 + .../pulsar-client-2x-shaded/pom.xml | 45 +- pulsar-client-admin-api/pom.xml | 134 +- pulsar-client-admin-shaded/pom.xml | 133 +- pulsar-client-admin/pom.xml | 26 +- pulsar-client-all/pom.xml | 79 +- pulsar-client-api/pom.xml | 145 +- .../src/main/resources/findbugsExclude.xml | 5 +- pulsar-client-auth-athenz/pom.xml | 22 +- .../src/test/resources/findbugsExclude.xml | 1 + pulsar-client-auth-sasl/pom.xml | 22 +- pulsar-client-messagecrypto-bc/pom.xml | 14 +- pulsar-client-shaded/pom.xml | 61 +- pulsar-client-tools-api/pom.xml | 12 +- .../pom.xml | 7 +- pulsar-client-tools-test/pom.xml | 15 +- .../src/test/resources/findbugsExclude.xml | 1 + pulsar-client-tools/pom.xml | 19 +- pulsar-client/pom.xml | 38 +- .../src/main/resources/findbugsExclude.xml | 1966 ++++++++--------- pulsar-common/pom.xml | 52 +- .../src/main/resources/findbugsExclude.xml | 66 +- pulsar-config-validation/pom.xml | 120 +- pulsar-functions/api-java/pom.xml | 30 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-functions/instance/pom.xml | 42 +- .../src/main/resources/findbugsExclude.xml | 1 + .../java-examples-builtin/pom.xml | 36 +- pulsar-functions/java-examples/pom.xml | 13 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-functions/localrun-shaded/pom.xml | 825 +++---- pulsar-functions/localrun/pom.xml | 157 +- .../src/main/resources/findbugsExclude.xml | 1 + .../localrun/src/main/resources/log4j2.xml | 24 +- pulsar-functions/pom.xml | 47 +- pulsar-functions/proto/pom.xml | 138 +- pulsar-functions/runtime-all/pom.xml | 25 +- .../src/main/resources/findbugsExclude.xml | 21 +- .../main/resources/java_instance_log4j2.xml | 219 +- .../resources/kubernetes_instance_log4j2.xml | 79 +- pulsar-functions/runtime/pom.xml | 21 +- .../src/main/resources/findbugsExclude.xml | 21 +- pulsar-functions/secrets/pom.xml | 13 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-functions/utils/pom.xml | 34 +- pulsar-functions/worker/pom.xml | 41 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-io/aerospike/pom.xml | 18 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-io/alluxio/pom.xml | 178 +- pulsar-io/aws/pom.xml | 17 +- .../src/main/resources/findbugsExclude.xml | 3 +- pulsar-io/batch-data-generator/pom.xml | 146 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/batch-discovery-triggerers/pom.xml | 16 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/canal/pom.xml | 216 +- pulsar-io/cassandra/pom.xml | 16 +- pulsar-io/common/pom.xml | 93 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/core/pom.xml | 12 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/data-generator/pom.xml | 88 +- pulsar-io/debezium/core/pom.xml | 23 +- pulsar-io/debezium/mongodb/pom.xml | 80 +- pulsar-io/debezium/mssql/pom.xml | 16 +- pulsar-io/debezium/mysql/pom.xml | 17 +- pulsar-io/debezium/oracle/pom.xml | 16 +- pulsar-io/debezium/pom.xml | 12 +- pulsar-io/debezium/postgres/pom.xml | 17 +- pulsar-io/docs/pom.xml | 14 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/dynamodb/pom.xml | 24 +- pulsar-io/elastic-search/pom.xml | 21 +- pulsar-io/file/pom.xml | 28 +- pulsar-io/flume/pom.xml | 290 ++- pulsar-io/hbase/pom.xml | 206 +- .../src/test/resources/hbase/hbase-site.xml | 41 +- pulsar-io/hdfs2/pom.xml | 124 +- .../src/main/resources/findbugsExclude.xml | 1 + .../src/test/resources/hadoop/core-site.xml | 17 +- .../src/test/resources/hadoop/hdfs-site.xml | 25 +- pulsar-io/hdfs3/pom.xml | 64 +- .../src/main/resources/findbugsExclude.xml | 1 + .../src/test/resources/hadoop/core-site.xml | 17 +- .../src/test/resources/hadoop/hdfs-site.xml | 25 +- pulsar-io/http/pom.xml | 119 +- pulsar-io/influxdb/pom.xml | 138 +- pulsar-io/jdbc/clickhouse/pom.xml | 16 +- pulsar-io/jdbc/core/pom.xml | 21 +- pulsar-io/jdbc/mariadb/pom.xml | 13 +- pulsar-io/jdbc/openmldb/pom.xml | 12 +- pulsar-io/jdbc/pom.xml | 10 +- pulsar-io/jdbc/postgres/pom.xml | 12 +- pulsar-io/jdbc/sqlite/pom.xml | 13 +- pulsar-io/kafka-connect-adaptor-nar/pom.xml | 37 +- pulsar-io/kafka-connect-adaptor/pom.xml | 36 +- pulsar-io/kafka/pom.xml | 25 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-io/kinesis/pom.xml | 35 +- pulsar-io/mongo/pom.xml | 16 +- pulsar-io/netty/pom.xml | 144 +- .../src/main/resources/findbugsExclude.xml | 25 +- pulsar-io/nsq/pom.xml | 29 +- pulsar-io/pom.xml | 16 +- pulsar-io/rabbitmq/pom.xml | 19 +- pulsar-io/redis/pom.xml | 149 +- pulsar-io/solr/pom.xml | 176 +- .../src/main/resources/findbugsExclude.xml | 45 +- pulsar-io/solr/src/test/resources/solr.xml | 34 +- pulsar-io/twitter/pom.xml | 26 +- pulsar-metadata/pom.xml | 27 +- .../src/main/resources/findbugsExclude.xml | 25 +- .../src/test/resources/findbugsExclude.xml | 19 +- .../bookkeeper-storage/pom.xml | 162 +- pulsar-package-management/core/pom.xml | 152 +- .../src/main/resources/findbugsExclude.xml | 1 + .../filesystem-storage/pom.xml | 61 +- pulsar-package-management/pom.xml | 74 +- pulsar-proxy/pom.xml | 38 +- pulsar-sql/pom.xml | 290 ++- pulsar-sql/presto-distribution/pom.xml | 35 +- .../src/assembly/assembly.xml | 121 +- pulsar-sql/presto-pulsar-plugin/pom.xml | 131 +- .../src/assembly/assembly.xml | 37 +- pulsar-sql/presto-pulsar/pom.xml | 442 ++-- pulsar-testclient/pom.xml | 303 ++- pulsar-transaction/common/pom.xml | 83 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-transaction/coordinator/pom.xml | 206 +- .../src/main/resources/findbugsExclude.xml | 1 + pulsar-transaction/pom.xml | 15 +- pulsar-websocket/pom.xml | 32 +- .../src/main/resources/findbugsExclude.xml | 4 +- src/assembly-source-package.xml | 10 +- src/check-binary-license.sh | 4 +- src/findbugs-exclude.xml | 28 +- src/idea-code-style.xml | 71 +- ...owasp-dependency-check-false-positives.xml | 76 +- src/owasp-dependency-check-suppressions.xml | 684 +++--- src/settings.xml | 3 +- structured-event-log/pom.xml | 14 +- testmocks/pom.xml | 19 +- tests/bc_2_0_0/pom.xml | 160 +- tests/bc_2_0_0/src/test/resources/pulsar.xml | 15 +- tests/bc_2_0_1/pom.xml | 160 +- tests/bc_2_0_1/src/test/resources/pulsar.xml | 15 +- tests/bc_2_6_0/pom.xml | 174 +- .../src/test/resources/backwards-client.xml | 15 +- .../docker-images/java-test-functions/pom.xml | 20 +- tests/docker-images/java-test-image/pom.xml | 17 +- tests/docker-images/java-test-plugins/pom.xml | 11 +- .../latest-version-image/pom.xml | 19 +- tests/docker-images/pom.xml | 10 +- tests/integration/pom.xml | 50 +- .../src/test/resources/pulsar-auth.xml | 9 +- .../pulsar-backwards-compatibility.xml | 29 +- .../src/test/resources/pulsar-cli.xml | 35 +- .../src/test/resources/pulsar-function.xml | 19 +- .../test/resources/pulsar-io-ora-source.xml | 13 +- .../src/test/resources/pulsar-io-sinks.xml | 15 +- .../src/test/resources/pulsar-io-sources.xml | 19 +- .../src/test/resources/pulsar-loadbalance.xml | 15 +- .../src/test/resources/pulsar-messaging.xml | 27 +- .../src/test/resources/pulsar-plugin.xml | 19 +- .../src/test/resources/pulsar-process.xml | 20 +- .../src/test/resources/pulsar-proxy.xml | 15 +- .../src/test/resources/pulsar-python.xml | 13 +- .../src/test/resources/pulsar-schema.xml | 17 +- .../src/test/resources/pulsar-semantics.xml | 13 +- .../src/test/resources/pulsar-sql.xml | 17 +- .../src/test/resources/pulsar-standalone.xml | 15 +- .../src/test/resources/pulsar-thread.xml | 22 +- .../src/test/resources/pulsar-tls.xml | 13 +- .../src/test/resources/pulsar-transaction.xml | 13 +- .../src/test/resources/pulsar-upgrade.xml | 13 +- .../src/test/resources/pulsar-websocket.xml | 15 +- .../integration/src/test/resources/pulsar.xml | 41 +- .../resources/tiered-filesystem-storage.xml | 15 +- .../test/resources/tiered-jcloud-storage.xml | 15 +- tests/pom.xml | 13 +- tests/pulsar-client-admin-shade-test/pom.xml | 187 +- .../src/test/resources/pulsar.xml | 15 +- tests/pulsar-client-all-shade-test/pom.xml | 185 +- .../src/test/resources/pulsar.xml | 15 +- tests/pulsar-client-shade-test/pom.xml | 175 +- .../src/test/resources/pulsar.xml | 15 +- tiered-storage/file-system/pom.xml | 383 ++-- .../src/main/resources/findbugsExclude.xml | 1 + tiered-storage/jcloud/pom.xml | 19 +- .../src/main/resources/findbugsExclude.xml | 1 + tiered-storage/pom.xml | 11 +- 228 files changed, 7254 insertions(+), 8374 deletions(-) diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 99fce8b2b9812..951f00c3cc3b8 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,4 +1,4 @@ - + - - - - - - + + + + + + diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 872764f899827..ca4639f8053e0 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -1,4 +1,4 @@ - + - + com.gradle gradle-enterprise-maven-extension diff --git a/.mvn/gradle-enterprise.xml b/.mvn/gradle-enterprise.xml index 2667402c23cdb..c8626080d0866 100644 --- a/.mvn/gradle-enterprise.xml +++ b/.mvn/gradle-enterprise.xml @@ -1,4 +1,4 @@ - + - + https://ge.apache.org false diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index a52fd8cec9b80..ef6637bac8917 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative bouncy-castle-parent - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - bouncy-castle-bc Apache Pulsar :: Bouncy Castle :: BC - ${project.groupId} @@ -39,23 +36,19 @@ ${project.version} provided - org.bouncycastle bcpkix-jdk18on ${bouncycastle.version} - org.bouncycastle bcprov-ext-jdk18on ${bouncycastle.version} - - de.ntcomputer @@ -73,7 +66,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 1298601c24e3e..19a2a2941bce1 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative bouncy-castle-parent - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - bcfips-include-test Pulsar Bouncy Castle FIPS Test Broker and client runs auth include BC FIPS verison - ${project.groupId} @@ -39,7 +37,6 @@ ${project.version} test - ${project.groupId} pulsar-broker @@ -53,7 +50,6 @@ test-jar test - ${project.groupId} pulsar-broker @@ -66,7 +62,6 @@ test - ${project.groupId} @@ -74,7 +69,6 @@ ${project.version} pkg - diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index 3fb9af4eb040d..f3d341ab6132c 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative bouncy-castle-parent - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - bouncy-castle-bcfips Apache Pulsar :: Bouncy Castle :: BC-FIPS - ${project.groupId} @@ -39,20 +36,17 @@ ${project.version} provided - org.bouncycastle bc-fips ${bouncycastle.bc-fips.version} - org.bouncycastle bcpkix-fips ${bouncycastle.bcpkix-fips.version} - diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index c8e5ff2f550b8..b94f33946df2b 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - @@ -42,16 +41,13 @@ - bouncy-castle-parent Apache Pulsar :: Bouncy Castle :: Parent - bc bcfips - bcfips-include-test diff --git a/buildtools/pom.xml b/buildtools/pom.xml index 4845ec7439bf8..afb431d6d837d 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache apache 29 - + - - org.apache.pulsar + io.streamnative buildtools - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT jar Pulsar Build Tools - - 2023-12-01T05:56:29Z + 2024-03-04T21:08:12Z 1.8 1.8 3.1.0 @@ -60,7 +57,6 @@ --add-opens java.base/jdk.internal.platform=ALL-UNNAMED - @@ -72,9 +68,7 @@ - - org.yaml snakeyaml @@ -95,7 +89,6 @@ guice ${guice.version} - org.testng testng @@ -147,7 +140,6 @@ ${mockito.version} - @@ -256,6 +248,55 @@ **/proto/* + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + + + maven-gpg-plugin + + + sign-artifacts + verify + + sign + + + + @@ -265,4 +306,14 @@ + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml index 184f58487eaf0..9e12d754fe7b5 100644 --- a/buildtools/src/main/resources/log4j2.xml +++ b/buildtools/src/main/resources/log4j2.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + diff --git a/buildtools/src/main/resources/pulsar/checkstyle.xml b/buildtools/src/main/resources/pulsar/checkstyle.xml index b3812ca8cccd7..14c8933540ac0 100644 --- a/buildtools/src/main/resources/pulsar/checkstyle.xml +++ b/buildtools/src/main/resources/pulsar/checkstyle.xml @@ -1,4 +1,4 @@ - + - - + - - - - - - - - - - - - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + - - - - + IMPORT CHECKS - - - - + --> + + + + - - - - - + + + + + + + - - - - + + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + } else --> - else --> - - - - - - - - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - - - - + + + diff --git a/buildtools/src/main/resources/pulsar/suppressions.xml b/buildtools/src/main/resources/pulsar/suppressions.xml index 7c78988db3e90..18a4e64d8836e 100644 --- a/buildtools/src/main/resources/pulsar/suppressions.xml +++ b/buildtools/src/main/resources/pulsar/suppressions.xml @@ -1,4 +1,4 @@ - + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/filesystem_offload_core_site.xml b/conf/filesystem_offload_core_site.xml index d26cec2cc60f0..3c7e6d94bbe50 100644 --- a/conf/filesystem_offload_core_site.xml +++ b/conf/filesystem_offload_core_site.xml @@ -1,3 +1,4 @@ + - - - fs.defaultFS - - - - hadoop.tmp.dir - pulsar - - - io.file.buffer.size - 4096 - - - io.seqfile.compress.blocksize - 1000000 - - - io.seqfile.compression.type - BLOCK - - - io.map.index.interval - 128 - - + + + fs.defaultFS + + + + hadoop.tmp.dir + pulsar + + + io.file.buffer.size + 4096 + + + io.seqfile.compress.blocksize + 1000000 + + + io.seqfile.compression.type + BLOCK + + + io.map.index.interval + 128 + diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index fd4042e82e82f..a1dcc6a1f654a 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-instance - 30 - - - pulsar.log.appender - RollingFile - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - RollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - ${sys:pulsar.function.log.file}*log.gz - - - 30d - - - - - - BkRollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - ${sys:pulsar.function.log.file}.bk*log.gz - - - 30d - - - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - BkRollingFile - - - - ${sys:pulsar.log.level} - - ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} - - - + pulsar-functions-instance + 30 + + + pulsar.log.appender + RollingFile + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + RollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + ${sys:pulsar.function.log.file}*log.gz + + + 30d + + + + + + BkRollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + ${sys:pulsar.function.log.file}.bk*log.gz + + + 30d + + + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + BkRollingFile + + + + ${sys:pulsar.log.level} + + ${sys:pulsar.log.appender} + ${sys:pulsar.log.level} + + + diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index 5f943d6f328d8..47d6b06a2bf8a 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-io-distribution pom Pulsar :: Distribution :: IO - ${project.groupId} @@ -46,7 +43,6 @@ provided - @@ -69,9 +65,32 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - generator-connector-config @@ -151,5 +170,4 @@ - diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 5b652170fdbb5..71d5aaf69f76a 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -1,3 +1,4 @@ + - + bin dir @@ -38,48 +37,120 @@ . 644 - - - ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar - ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar - ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar - ${basedir}/../../pulsar-io/http/target/pulsar-io-http-${project.version}.nar - ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar - ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar - ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/sqlite/target/pulsar-io-jdbc-sqlite-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/mariadb/target/pulsar-io-jdbc-mariadb-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/clickhouse/target/pulsar-io-jdbc-clickhouse-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/postgres/target/pulsar-io-jdbc-postgres-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/openmldb/target/pulsar-io-jdbc-openmldb-${project.version}.nar - ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar - ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar - ${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar - ${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar - ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar - ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar - ${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar - ${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar - ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar - ${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar - ${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar - ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar - ${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar - ${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar - ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar - ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar - ${basedir}/../../pulsar-io/alluxio/target/pulsar-io-alluxio-${project.version}.nar + + ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar + + + ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar + + + ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar + + + ${basedir}/../../pulsar-io/http/target/pulsar-io-http-${project.version}.nar + + + ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar + + + ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar + + + ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/sqlite/target/pulsar-io-jdbc-sqlite-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/mariadb/target/pulsar-io-jdbc-mariadb-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/clickhouse/target/pulsar-io-jdbc-clickhouse-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/postgres/target/pulsar-io-jdbc-postgres-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/openmldb/target/pulsar-io-jdbc-openmldb-${project.version}.nar + + + ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar + + + ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar + + + ${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar + + + ${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar + + + ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar + + + ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar + + + ${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar + + + ${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar + + + ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar + + + ${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar + + + ${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar + + + ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar + + + ${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar + + + ${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar + + + ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar + + + ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar + + + ${basedir}/../../pulsar-io/alluxio/target/pulsar-io-alluxio-${project.version}.nar + diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index ebc069838e09e..b6058a3570dc1 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-offloader-distribution pom Pulsar :: Distribution :: Offloader - ${project.groupId} @@ -66,7 +63,6 @@ - @@ -90,6 +86,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/offloaders/src/assemble/offloaders.xml b/distribution/offloaders/src/assemble/offloaders.xml index 38f7eee906064..19c60f68c21d4 100644 --- a/distribution/offloaders/src/assemble/offloaders.xml +++ b/distribution/offloaders/src/assemble/offloaders.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -38,13 +37,11 @@ . 644 - ${basedir}/../../tiered-storage/jcloud/target/tiered-storage-jcloud-${project.version}.nar offloaders 644 - ${basedir}/../../tiered-storage/file-system/target/tiered-storage-file-system-${project.version}.nar offloaders diff --git a/distribution/pom.xml b/distribution/pom.xml index 08eabeec53d11..90ed0bf4a57cb 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - distribution pom Pulsar :: Distribution - - main @@ -47,7 +43,6 @@ shell - core-modules @@ -55,7 +50,6 @@ - @@ -65,6 +59,30 @@ ${skipBuildDistribution} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index b51bae3177750..e32c0dfdf05a0 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-server-distribution pom Pulsar :: Distribution :: Server - ${project.groupId} pulsar-broker ${project.version} - ${project.groupId} pulsar-proxy ${project.version} - ${project.groupId} pulsar-broker-auth-oidc ${project.version} - ${project.groupId} pulsar-broker-auth-sasl ${project.version} - ${project.groupId} pulsar-client-auth-sasl ${project.version} - jline jline ${jline.version} - org.apache.zookeeper zookeeper-prometheus-metrics @@ -89,7 +80,6 @@ - ${project.groupId} pulsar-package-bookkeeper-storage @@ -101,13 +91,11 @@ - ${project.groupId} pulsar-package-filesystem-storage ${project.version} - ${project.groupId} pulsar-client-tools @@ -119,33 +107,27 @@ - ${project.groupId} pulsar-testclient ${project.version} - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - org.apache.logging.log4j log4j-web - io.dropwizard.metrics metrics-core - io.dropwizard.metrics metrics-graphite @@ -156,44 +138,36 @@ - io.dropwizard.metrics metrics-jvm - org.xerial.snappy snappy-java - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - org.apache.logging.log4j log4j-slf4j-impl - org.apache.bookkeeper.stats prometheus-metrics-provider - io.prometheus simpleclient_log4j2 - ${project.groupId} bouncy-castle-bc ${project.version} pkg - ${project.groupId} @@ -208,7 +182,6 @@ - ${project.groupId} pulsar-functions-worker @@ -225,7 +198,6 @@ - ${project.groupId} @@ -240,7 +212,6 @@ - ${project.groupId} @@ -253,7 +224,6 @@ - io.grpc @@ -263,13 +233,11 @@ org.bouncycastle bcpkix-jdk18on - io.perfmark perfmark-api compile - org.apache.bookkeeper.http @@ -291,7 +259,6 @@ vertx-web - @@ -332,6 +299,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml index 41ac24d0582da..f65a8e2fb7260 100644 --- a/distribution/server/src/assemble/bin.xml +++ b/distribution/server/src/assemble/bin.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -131,14 +130,11 @@ ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} - - org.apache.pulsar:pulsar-functions-runtime-all - + io.streamnative:pulsar-functions-runtime-all org.projectlombok:lombok - - org.apache.pulsar:pulsar-functions-api-examples + io.streamnative:pulsar-functions-api-examples *:tar.gz diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml index 5a1df2aec4421..d07a444115cd5 100644 --- a/distribution/shell/pom.xml +++ b/distribution/shell/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-shell-distribution pom Pulsar :: Distribution :: Shell - ${project.groupId} pulsar-client-tools ${project.version} - org.apache.logging.log4j log4j-core - org.apache.logging.log4j log4j-web @@ -53,14 +48,11 @@ org.apache.logging.log4j log4j-slf4j-impl - io.prometheus simpleclient_log4j2 - - @@ -101,6 +93,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/shell/src/assemble/shell.xml b/distribution/shell/src/assemble/shell.xml index f823e0258b231..cd11bd8a1ef57 100644 --- a/distribution/shell/src/assemble/shell.xml +++ b/distribution/shell/src/assemble/shell.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -46,7 +45,6 @@ . 644 - bin ${basedir}/../../bin/pulsar-admin-common.sh @@ -76,7 +74,6 @@ ${basedir}/../../conf/log4j2.yaml - lib diff --git a/docker/pom.xml b/docker/pom.xml index 8ff2a8ae7ebba..e6e168f65349a 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT docker-images Apache Pulsar :: Docker Images @@ -39,6 +38,30 @@ ${skipBuildDistribution} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 04f01272b7afb..b9f01f6189008 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar + io.streamnative docker-images - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 pulsar-all-docker-image Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All Components) pom - ${project.groupId} @@ -64,7 +63,6 @@ - git-commit-id-no-git @@ -177,7 +175,6 @@ - docker-push @@ -200,6 +197,5 @@ - diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 413cb3cd0671f..246b9a07e309c 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar + io.streamnative docker-images - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 pulsar-docker-image Apache Pulsar :: Docker Images :: Pulsar Latest Version pom - ${project.groupId} @@ -46,12 +45,10 @@ - mirror://mirrors.ubuntu.com/mirrors.txt http://security.ubuntu.com/ubuntu/ - git-commit-id-no-git @@ -128,7 +125,6 @@ - docker-push @@ -151,6 +147,5 @@ - diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index e9a63d5d686e5..3b028bb8afc73 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - jclouds-shaded Apache Pulsar :: Jclouds shaded - - org.apache.jclouds jclouds-allblobstore @@ -66,7 +61,6 @@ javax.annotation-api - @@ -82,7 +76,6 @@ true true false - com.google.guava:guava @@ -106,7 +99,6 @@ com.google.errorprone:* - com.google @@ -140,21 +132,20 @@ com.google.errorprone org.apache.pulsar.jcloud.shade.com.google.errorprone - - - org.apache.jclouds:jclouds-core - - - lib/gson*jar - - - + + org.apache.jclouds:jclouds-core + + + lib/gson*jar + + + diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 8a1078fcc2e27..fe1cf0cb3a2ae 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - managed-ledger Managed Ledger - org.apache.bookkeeper bookkeeper-server - org.apache.bookkeeper.stats prometheus-metrics-provider - org.apache.bookkeeper.stats codahale-metrics-provider @@ -54,36 +49,30 @@ - com.google.protobuf protobuf-java - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-metadata ${project.version} - com.google.guava guava - ${project.groupId} testmocks ${project.version} test - org.apache.zookeeper zookeeper @@ -105,29 +94,25 @@ - io.dropwizard.metrics - metrics-core - test + io.dropwizard.metrics + metrics-core + test - org.xerial.snappy - snappy-java - test + org.xerial.snappy + snappy-java + test - org.awaitility awaitility test - org.slf4j slf4j-api - - @@ -147,7 +132,6 @@ - org.apache.maven.plugins maven-jar-plugin diff --git a/pom.xml b/pom.xml index 122b551d0e818..375087e4f6a54 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 pom @@ -28,30 +27,24 @@ apache 29 - - org.apache.pulsar + io.streamnative pulsar - - 3.0.3-SNAPSHOT - + 3.0.0-SNAPSHOT Pulsar Pulsar is a distributed pub-sub messaging platform with a very flexible messaging model and an intuitive client API. https://github.com/apache/pulsar - Apache Software Foundation https://www.apache.org/ 2017 - Apache Pulsar developers https://pulsar.apache.org/ - Apache License, Version 2.0 @@ -59,47 +52,38 @@ flexible messaging model and an intuitive client API. repo - - https://github.com/apache/pulsar - scm:git:https://github.com/apache/pulsar.git - scm:git:ssh://git@github.com:apache/pulsar.git + https://github.com/streamnative/pulsar + scm:git:https://github.com/streamnative/pulsar.git + scm:git:ssh://git@github.com:streamnative/pulsar.git - GitHub Actions https://github.com/apache/pulsar/actions - Github https://github.com/apache/pulsar/issues - 17 17 ${maven.compiler.target} 8 - 3.4.0 - **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java quarantine - UTF-8 UTF-8 - 2023-12-01T05:56:29Z + 2024-03-04T21:08:12Z true - - - + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED @@ -116,7 +100,7 @@ flexible messaging model and an intuitive client API. false 1 true - + false ${project.build.directory} @@ -129,16 +113,16 @@ flexible messaging model and an intuitive client API. false package package - 1.26.0 - 4.16.4 3.9.1 1.5.0 1.10.0 - 1.1.10.5 - 4.1.12.1 + 1.1.10.5 + + 4.1.12.1 + 5.1.0 4.1.100.Final 0.0.21.Final @@ -251,11 +235,9 @@ flexible messaging model and an intuitive client API. 3.4.3 1.5.2-3 2.0.6 - 1.18.3 2.2 - 3.3.0 1.1.1 @@ -269,7 +251,6 @@ flexible messaging model and an intuitive client API. 1.5.4 5.4.0 2.33.2 - 0.6.1 3.0.0 @@ -303,20 +284,16 @@ flexible messaging model and an intuitive client API. 0.9.44 1.6.1 6.4.0 - rename-netty-native-libs.sh - - org.jline jline ${jline3.version} - org.asynchttpclient async-http-client @@ -332,45 +309,39 @@ flexible messaging model and an intuitive client API. - org.testng testng ${testng.version} - - org.yaml - * - + + org.yaml + * + - org.hamcrest hamcrest ${hamcrest.version} test - org.awaitility awaitility ${awaitility.version} test - org.mockito mockito-core ${mockito.version} - org.mockito mockito-inline ${mockito.version} - org.apache.zookeeper zookeeper @@ -446,7 +417,6 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper bookkeeper-server @@ -475,13 +445,11 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper cpu-affinity ${bookkeeper.version} - io.vertx vertx-core @@ -492,38 +460,33 @@ flexible messaging model and an intuitive client API. vertx-web ${vertx.version} - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.zookeeper - * - - + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.zookeeper + * + + - org.apache.bookkeeper bookkeeper-common-allocator ${bookkeeper.version} - org.apache.bookkeeper bookkeeper-tools-framework ${bookkeeper.version} - org.reflections reflections ${reflections.version} - org.apache.bookkeeper @@ -556,7 +519,6 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper @@ -589,19 +551,16 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper bookkeeper-common ${bookkeeper.version} - org.apache.bookkeeper.stats bookkeeper-stats-api ${bookkeeper.version} - org.apache.bookkeeper.stats datasketches-metrics-provider @@ -613,37 +572,31 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper.stats prometheus-metrics-provider ${bookkeeper.version} - org.rocksdb rocksdbjni ${rocksdb.version} - org.eclipse.jetty jetty-server ${jetty.version} - org.eclipse.jetty jetty-alpn-conscrypt-server ${jetty.version} - org.conscrypt conscrypt-openjdk-uber ${conscrypt.version} - org.eclipse.jetty jetty-bom @@ -651,7 +604,6 @@ flexible messaging model and an intuitive client API. pom import - io.netty netty-bom @@ -659,7 +611,6 @@ flexible messaging model and an intuitive client API. pom import - io.netty.incubator netty-incubator-transport-classes-io_uring @@ -682,79 +633,66 @@ flexible messaging model and an intuitive client API. ${netty-iouring.version} linux-aarch_64 - com.beust jcommander ${jcommander.version} - com.google.guava guava ${guava.version} - com.google.inject guice ${guice.version} - com.google.inject.extensions guice-assistedinject ${guice.version} - org.apache.commons commons-lang3 ${commons-lang3.version} - org.apache.commons commons-compress ${commons-compress.version} - commons-configuration commons-configuration ${commons-configuration.version} - commons-io commons-io ${commons-io.version} - org.apache.commons commons-text ${commons-text.version} - org.slf4j slf4j-api ${slf4j.version} - org.slf4j slf4j-simple ${slf4j.version} - org.slf4j jcl-over-slf4j ${slf4j.version} - org.apache.logging.log4j log4j-bom @@ -762,49 +700,41 @@ flexible messaging model and an intuitive client API. pom import - commons-codec commons-codec ${commons-codec.version} - org.glassfish.jersey.core jersey-server ${jersey.version} - org.glassfish.jersey.core jersey-client ${jersey.version} - org.glassfish.jersey.inject jersey-hk2 ${jersey.version} - org.glassfish.jersey.containers jersey-container-servlet-core ${jersey.version} - org.glassfish.jersey.containers jersey-container-servlet ${jersey.version} - javax.ws.rs javax.ws.rs-api ${javax.ws.rs-api.version} - org.glassfish.jersey.media jersey-media-json-jackson @@ -816,19 +746,16 @@ flexible messaging model and an intuitive client API. - org.glassfish.jersey.media jersey-media-multipart ${jersey.version} - net.java.dev.jna jna ${jna.version} - com.github.docker-java docker-java-core @@ -850,7 +777,6 @@ flexible messaging model and an intuitive client API. docker-java-transport-zerodep ${docker-java.version} - com.fasterxml.jackson jackson-bom @@ -858,56 +784,46 @@ flexible messaging model and an intuitive client API. pom import - org.codehaus.jettison jettison ${jettison.version} - com.fasterxml.woodstox woodstox-core ${woodstox.version} - - org.hdrhistogram HdrHistogram ${hdrHistogram.version} - io.swagger swagger-core ${swagger.version} - io.swagger swagger-annotations ${swagger.version} - javax.servlet javax.servlet-api ${javax.servlet-api} - com.github.ben-manes.caffeine caffeine ${caffeine.version} - org.bouncycastle bcpkix-jdk18on ${bouncycastle.version} - com.cronutils cron-utils @@ -919,19 +835,16 @@ flexible messaging model and an intuitive client API. - com.yahoo.athenz athenz-zts-java-client-core ${athenz.version} - com.yahoo.athenz athenz-zpe-java-client ${athenz.version} - com.yahoo.athenz athenz-cert-refresher @@ -943,7 +856,6 @@ flexible messaging model and an intuitive client API. - com.yahoo.athenz athenz-auth-core @@ -955,67 +867,56 @@ flexible messaging model and an intuitive client API. - com.github.zafarkhaja java-semver ${java-semver.version} - io.prometheus simpleclient ${prometheus.version} - io.prometheus simpleclient_hotspot ${prometheus.version} - io.prometheus simpleclient_log4j2 ${prometheus.version} - io.prometheus simpleclient_servlet ${prometheus.version} - io.prometheus simpleclient_jetty ${prometheus.version} - io.prometheus simpleclient_caffeine ${prometheus.version} - com.carrotsearch hppc ${hppc.version} - io.etcd jetcd-core ${jetcd.version} - io.etcd jetcd-test ${jetcd.version} - org.apache.spark spark-streaming_2.10 @@ -1043,7 +944,6 @@ flexible messaging model and an intuitive client API. - io.jsonwebtoken jjwt-api @@ -1059,25 +959,21 @@ flexible messaging model and an intuitive client API. jjwt-jackson ${jsonwebtoken.version} - net.jodah typetools ${typetools.version} - net.bytebuddy byte-buddy ${byte-buddy.version} - org.zeroturnaround zt-zip ${zt-zip.version} - io.grpc grpc-bom @@ -1085,7 +981,6 @@ flexible messaging model and an intuitive client API. pom import - io.grpc grpc-all @@ -1109,7 +1004,6 @@ flexible messaging model and an intuitive client API. - io.grpc grpc-xds @@ -1121,25 +1015,21 @@ flexible messaging model and an intuitive client API. - com.google.http-client google-http-client ${google-http-client.version} - com.google.http-client google-http-client-jackson2 ${google-http-client.version} - com.google.http-client google-http-client-gson ${google-http-client.version} - io.perfmark perfmark-api @@ -1152,7 +1042,6 @@ flexible messaging model and an intuitive client API. - com.google.protobuf protobuf-bom @@ -1160,19 +1049,16 @@ flexible messaging model and an intuitive client API. pom import - com.google.code.gson gson ${gson.version} - com.yahoo.datasketches sketches-core ${sketches.version} - com.amazonaws aws-java-sdk-bom @@ -1180,7 +1066,6 @@ flexible messaging model and an intuitive client API. pom import - org.apache.distributedlog distributedlog-core @@ -1193,13 +1078,11 @@ flexible messaging model and an intuitive client API. - org.apache.commons commons-collections4 ${commons.collections4.version} - com.lmax @@ -1223,103 +1106,86 @@ flexible messaging model and an intuitive client API. assertj-core ${assertj-core.version} - org.projectlombok lombok ${lombok.version} - javax.annotation javax.annotation-api ${javax.annotation-api.version} - javax.xml.bind jaxb-api ${jaxb-api} - jakarta.xml.bind jakarta.xml.bind-api ${jakarta.xml.bind.version} - com.sun.activation javax.activation ${javax.activation.version} - com.sun.activation jakarta.activation ${jakarta.activation.version} - jakarta.activation jakarta.activation-api ${jakarta.activation.version} - jakarta.validation jakarta.validation-api ${jakarta.validation.version} - io.opencensus opencensus-api ${opencensus.version} - io.opencensus opencensus-contrib-http-util ${opencensus.version} - io.opencensus opencensus-contrib-grpc-metrics ${opencensus.version} - org.opensearch.client opensearch-rest-high-level-client ${opensearch.version} - co.elastic.clients elasticsearch-java ${elasticsearch-java.version} - joda-time joda-time ${joda.version} - org.javassist javassist ${javassist.version} - net.jcip jcip-annotations ${jcip.version} - io.airlift aircompressor @@ -1331,25 +1197,21 @@ flexible messaging model and an intuitive client API. - org.objenesis objenesis ${objenesis.version} - org.apache.httpcomponents httpclient ${apache-http-client.version} - org.apache.httpcomponents httpcore ${apache-httpcomponents.version} - com.github.spotbugs spotbugs-annotations @@ -1357,31 +1219,26 @@ flexible messaging model and an intuitive client API. provided true - com.google.errorprone error_prone_annotations ${errorprone.version} - com.google.j2objc j2objc-annotations ${j2objc-annotations.version} - org.yaml snakeyaml ${snakeyaml.version} - org.apache.ant ant ${ant.version} - com.squareup.okhttp3 okhttp @@ -1402,7 +1259,6 @@ flexible messaging model and an intuitive client API. okio ${okio.version} - org.jetbrains.kotlin kotlin-stdlib @@ -1413,31 +1269,26 @@ flexible messaging model and an intuitive client API. kotlin-stdlib-common ${kotlin-stdlib.version} - org.jetbrains.kotlin kotlin-stdlib-jdk8 ${kotlin-stdlib.version} - com.github.luben zstd-jni ${zstd-jni.version} - - com.typesafe.netty - netty-reactive-streams - ${netty-reactive-streams.version} + com.typesafe.netty + netty-reactive-streams + ${netty-reactive-streams.version} - ch.qos.reload4j reload4j ${reload4j.version} - org.roaringbitmap RoaringBitmap @@ -1450,47 +1301,40 @@ flexible messaging model and an intuitive client API. - - org.apache.pulsar + io.streamnative buildtools ${project.version} test - org.testng testng test - org.mockito mockito-core test - org.mockito mockito-inline test - com.github.stefanbirkner system-lambda ${system-lambda.version} test - org.assertj assertj-core test - org.projectlombok lombok @@ -1501,7 +1345,6 @@ flexible messaging model and an intuitive client API. javax.annotation-api provided - org.apache.bookkeeper @@ -1511,8 +1354,8 @@ flexible messaging model and an intuitive client API. tests - org.bouncycastle - * + org.bouncycastle + * org.slf4j @@ -1546,8 +1389,12 @@ flexible messaging model and an intuitive client API. + + org.apache.logging.log4j + log4j-layout-template-json + 2.20.0 + - ${project.artifactId} @@ -1622,7 +1469,6 @@ flexible messaging model and an intuitive client API. - org.commonjava.maven.plugins directory-maven-plugin @@ -1637,14 +1483,13 @@ flexible messaging model and an intuitive client API. pulsar.basedir - org.apache.pulsar + io.streamnative pulsar - pl.project13.maven git-commit-id-plugin @@ -1669,7 +1514,6 @@ flexible messaging model and an intuitive client API. - com.mycila license-maven-plugin @@ -1770,33 +1614,26 @@ flexible messaging model and an intuitive client API. src/assemble/README.bin.txt src/assemble/LICENSE.bin.txt src/assemble/NOTICE.bin.txt - src/main/java/org/apache/bookkeeper/mledger/proto/MLDataFormats.java src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java bin/proto/MLDataFormats_pb2.py - **/avro/generated/*.java - **/*.avsc - src/main/java/org/apache/pulsar/io/kinesis/fbs/CompressionType.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionCtx.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionKey.java src/main/java/org/apache/pulsar/io/kinesis/fbs/KeyValue.java src/main/java/org/apache/pulsar/io/kinesis/fbs/Message.java - src/main/java/org/apache/bookkeeper/mledger/util/AbstractCASReferenceCounted.java - dependency-reduced-pom.xml - pulsar-client-go/go.mod pulsar-client-go/go.sum @@ -1804,15 +1641,12 @@ flexible messaging model and an intuitive client API. pulsar-function-go/go.sum pulsar-function-go/examples/go.mod pulsar-function-go/examples/go.sum - **/META-INF/services/com.scurrilous.circe.HashProvider **/META-INF/services/io.trino.spi.Plugin - **/django/stats/migrations/*.py **/conf/uwsgi_params - **/*.crt **/*.key @@ -1825,21 +1659,16 @@ flexible messaging model and an intuitive client API. certificate-authority/index.txt certificate-authority/serial certificate-authority/README.md - **/zk-3.5-test-data/* - **/requirements.txt - conf/schema_example.json **/templates/*.tpl - **/.helmignore **/_helpers.tpl - **/*.md .github/** @@ -1868,6 +1697,7 @@ flexible messaging model and an intuitive client API. **/*.so.* **/*.dylib src/test/resources/*.txt + **/dependency-reduced-pom.xml @@ -1876,55 +1706,106 @@ flexible messaging model and an intuitive client API. maven-enforcer-plugin ${maven-enforcer-plugin.version} - - enforce-maven - - enforce - - - - - 17 - Java 17+ is required to build Pulsar. - - - 3.6.1 - - - - + + enforce-maven + + enforce + + + + + 17 + Java 17+ is required to build Pulsar. + + + 3.6.1 + + + + - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - false - - - source-release-assembly-tar-gz - generate-sources - - single - - - ${skipSourceReleaseAssembly} - true - - src/assembly-source-package.xml - - apache-pulsar-${project.version}-src - false - - tar.gz - - posix - - - + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + false + + + source-release-assembly-tar-gz + generate-sources + + single + + + ${skipSourceReleaseAssembly} + true + + src/assembly-source-package.xml + + apache-pulsar-${project.version}-src + false + + tar.gz + + posix + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + true + + ossrh + https://s01.oss.sonatype.org/ + 60285aee9d4161 + true + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + none + false + + + + + + org.apache.maven.plugins + maven-release-plugin + 3.0.0-M1 + + true + false + release + deploy + - @@ -1945,10 +1826,10 @@ flexible messaging model and an intuitive client API. maven-surefire-plugin - ${include} + ${include} - **/*$*,${exclude} + **/*$*,${exclude} ${groups} ${excludedGroups} @@ -2032,13 +1913,13 @@ flexible messaging model and an intuitive client API. com.github.spotbugs spotbugs-maven-plugin ${spotbugs-maven-plugin.version} - - - com.github.spotbugs - spotbugs - ${spotbugs.version} - - + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + org.codehaus.mojo @@ -2050,6 +1931,16 @@ flexible messaging model and an intuitive client API. docker-maven-plugin ${docker-maven.version} + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + @@ -2065,7 +1956,6 @@ flexible messaging model and an intuitive client API. - @@ -2174,7 +2064,6 @@ flexible messaging model and an intuitive client API. tests - contrib-check - - - - org.apache.rat - apache-rat-plugin - - - - check - - verify - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - check-style - verify - - ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml - ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml - UTF-8 - **/proto/* - - - check - - - - - - + + + + org.apache.rat + apache-rat-plugin + + + + check + + verify + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + check-style + verify + + ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml + ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml + UTF-8 + **/proto/* + + + check + + + + + + - windows @@ -2230,7 +2118,6 @@ flexible messaging model and an intuitive client API. rename-netty-native-libs.cmd - @@ -2274,37 +2161,27 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation - structured-event-log - pulsar-transaction - pulsar-functions - pulsar-io - bouncy-castle - pulsar-client-messagecrypto-bc - pulsar-metadata jclouds-shaded - pulsar-package-management - distribution docker tests - core-modules @@ -2333,31 +2210,23 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation - pulsar-transaction - pulsar-functions - pulsar-io - bouncy-castle - pulsar-client-messagecrypto-bc - distribution pulsar-metadata - pulsar-package-management - - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-broker-auth-athenz jar Athenz authentication plugin for broker - - ${project.groupId} pulsar-broker ${project.version} - ${project.groupId} testmocks ${project.version} test - com.yahoo.athenz athenz-zpe-java-client - org.bouncycastle bcpkix-jdk18on - - @@ -79,7 +69,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -97,7 +86,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -111,7 +99,6 @@ - diff --git a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml +++ b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-broker-auth-oidc/pom.xml b/pulsar-broker-auth-oidc/pom.xml index 2f5edbb129a0e..0e2da4aa0861d 100644 --- a/pulsar-broker-auth-oidc/pom.xml +++ b/pulsar-broker-auth-oidc/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-broker-auth-oidc jar Open ID Connect authentication plugin for broker - 0.11.5 - - ${project.groupId} pulsar-broker-common @@ -50,29 +44,24 @@ - com.auth0 java-jwt 4.3.0 - com.auth0 jwks-rsa 0.22.0 - com.github.ben-manes.caffeine caffeine - org.asynchttpclient async-http-client - io.kubernetes client-java @@ -96,9 +85,7 @@ org.bouncycastle - - io.jsonwebtoken jjwt-api @@ -111,16 +98,13 @@ ${jsonwebtoken.version} test - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - - @@ -142,8 +126,6 @@ - - @@ -163,7 +145,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index d627cced58e3c..87fcbf2f7db56 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-broker-auth-sasl jar SASL authentication plugin for broker - - ${project.groupId} pulsar-broker ${project.version} - org.apache.kerby kerby-config @@ -53,7 +47,6 @@ - org.apache.kerby kerb-simplekdc @@ -66,30 +59,25 @@ - ${project.groupId} pulsar-proxy ${project.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-client-auth-sasl ${project.version} test - - @@ -111,8 +99,6 @@ - - @@ -132,7 +118,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index 02fd389be91fb..44b25ab0cf3e4 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-broker-common Common classes used in multiple broker modules - ${project.groupId} pulsar-metadata ${project.version} - com.google.guava guava - io.prometheus simpleclient_jetty - javax.servlet javax.servlet-api - javax.ws.rs javax.ws.rs-api - io.jsonwebtoken jjwt-impl - io.jsonwebtoken jjwt-jackson - org.bouncycastle @@ -76,14 +65,12 @@ ${bouncycastle.bc-fips.version} test - org.awaitility awaitility test - @@ -103,7 +90,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -117,7 +103,6 @@ - maven-resources-plugin diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 8545dba755357..19165c87ffc39 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-broker jar Pulsar Broker - commons-codec commons-codec - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - org.slf4j slf4j-api - io.netty netty-transport - com.google.protobuf protobuf-java - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-websocket ${project.version} - ${project.groupId} pulsar-client-admin-original ${project.version} - ${project.groupId} managed-ledger ${project.version} - org.apache.curator curator-recipes - org.apache.bookkeeper stream-storage-server @@ -119,241 +105,196 @@ - org.apache.bookkeeper bookkeeper-tools-framework - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-transaction-common ${project.version} - ${project.groupId} pulsar-io-batch-discovery-triggerers ${project.version} test - ${project.groupId} testmocks ${project.version} test - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - - io.dropwizard.metrics - metrics-core + io.dropwizard.metrics + metrics-core - - org.xerial.snappy - snappy-java + org.xerial.snappy + snappy-java - - ${project.groupId} pulsar-functions-worker ${project.version} - ${project.groupId} pulsar-functions-local-runner-original ${project.version} test - ${project.groupId} pulsar-client-messagecrypto-bc ${project.version} - org.awaitility awaitility test - - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.media jersey-media-json-jackson - org.glassfish.jersey.test-framework jersey-test-framework-core test ${jersey.version} - org.glassfish.jersey.test-framework.providers jersey-test-framework-provider-grizzly2 test ${jersey.version} - jakarta.activation jakarta.activation-api - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - com.fasterxml.jackson.module jackson-module-jsonSchema - org.slf4j jcl-over-slf4j - com.google.guava guava - com.beust jcommander - io.swagger swagger-annotations - io.prometheus simpleclient - io.prometheus simpleclient_jetty - io.prometheus simpleclient_hotspot - io.prometheus simpleclient_caffeine - io.swagger swagger-core - org.hdrhistogram HdrHistogram - com.google.code.gson gson - com.github.zafarkhaja java-semver - org.apache.avro avro ${avro.version} - com.carrotsearch hppc - org.roaringbitmap RoaringBitmap - com.github.oshi oshi-core-java11 - ${project.groupId} pulsar-functions-api-examples @@ -361,7 +302,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples-builtin @@ -369,7 +309,6 @@ pom test - ${project.groupId} pulsar-io-batch-data-generator @@ -377,7 +316,6 @@ pom test - ${project.groupId} pulsar-io-data-generator @@ -385,7 +323,6 @@ pom test - ${project.groupId} pulsar-metadata @@ -393,7 +330,6 @@ test-jar test - javax.xml.bind jaxb-api @@ -404,42 +340,33 @@ - com.sun.activation javax.activation - - ${project.groupId} pulsar-transaction-coordinator ${project.version} - - ${project.groupId} pulsar-package-core ${project.version} - io.etcd jetcd-test test - ${project.groupId} pulsar-package-filesystem-storage ${project.version} - - @@ -462,7 +389,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -493,7 +419,6 @@ - maven-dependency-plugin @@ -546,7 +471,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -562,7 +486,6 @@ - org.xolstice.maven.plugins protobuf-maven-plugin @@ -632,7 +555,6 @@ - maven-resources-plugin @@ -663,7 +585,6 @@ - @@ -682,7 +603,6 @@ test-jar test - ${project.groupId} pulsar-package-core @@ -690,7 +610,6 @@ test-jar test - ${project.groupId} pulsar-metadata diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 30fd3259252ef..ebfc492578599 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-1x-base Pulsar Client 1.x Compatibility Base pom - pulsar-client-2x-shaded pulsar-client-1x - @@ -61,7 +57,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -77,5 +72,4 @@ - diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index b7f1bb5f37808..571e7cfebb96e 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-client-1x-base - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-1x Pulsar Client 1.x Compatibility API - ${project.groupId} pulsar-client-2x-shaded ${project.version} - com.google.guava guava - org.apache.commons commons-lang3 - - @@ -70,7 +63,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -90,5 +82,4 @@ - diff --git a/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml b/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml index 7938e60bf4330..d191e191d52ca 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-client-1x-base - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-2x-shaded Pulsar Client 2.x Shaded API - ${project.groupId} @@ -39,10 +36,9 @@ ${project.version} - - + org.apache.maven.plugins maven-shade-plugin @@ -56,16 +52,15 @@ true true false - - org.apache.pulsar:pulsar-client - org.apache.pulsar:pulsar-client-api + io.streamnative:pulsar-client + io.streamnative:pulsar-client-api - org.apache.pulsar:pulsar-client + io.streamnative:pulsar-client ** @@ -93,6 +88,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index 2081918f857ea..f90b1a21bc439 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.0.3-SNAPSHOT - .. - - - pulsar-client-admin-api - Pulsar Client Admin :: API - - - - ${project.groupId} - pulsar-client-api - ${project.version} - - - - org.slf4j - slf4j-api - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar + 3.0.0-SNAPSHOT + .. + + pulsar-client-admin-api + Pulsar Client Admin :: API + + + ${project.groupId} + pulsar-client-api + ${project.version} + + + org.slf4j + slf4j-api + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index b8132c48df828..80df41d833f3d 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-admin Pulsar Client Admin - ${project.groupId} @@ -74,7 +70,6 @@ - maven-antrun-plugin @@ -87,15 +82,12 @@ - + - org.apache.maven.plugins maven-shade-plugin @@ -108,11 +100,10 @@ true true - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec commons-collections:commons-collections @@ -126,7 +117,7 @@ com.fasterxml.jackson.*:* io.netty:* io.netty.incubator:* - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:* com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* @@ -148,7 +139,7 @@ org.yaml:snakeyaml io.swagger:* - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -156,7 +147,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -166,7 +157,7 @@ - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-admin-original ** @@ -177,7 +168,7 @@ - + org.asynchttpclient org.apache.pulsar.shade.org.asynchttpclient @@ -259,51 +250,75 @@ org.reactivestreams org.apache.pulsar.shade.org.reactivestreams - - io.grpc - org.apache.pulsar.shade.io.grpc - - - okio - org.apache.pulsar.shade.okio - - - com.squareup - org.apache.pulsar.shade.com.squareup - - - io.opencensus - org.apache.pulsar.shade.io.opencensus - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - org.objenesis - org.apache.pulsar.shade.org.objenesis - - - org.yaml - org.apache.pulsar.shade.org.yaml - - - io.swagger - org.apache.pulsar.shade.io.swagger - - - org.apache.bookkeeper - org.apache.pulsar.shade.org.apache.bookkeeper - + + io.grpc + org.apache.pulsar.shade.io.grpc + + + okio + org.apache.pulsar.shade.okio + + + com.squareup + org.apache.pulsar.shade.com.squareup + + + io.opencensus + org.apache.pulsar.shade.io.opencensus + + + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty + + + org.objenesis + org.apache.pulsar.shade.org.objenesis + + + org.yaml + org.apache.pulsar.shade.org.yaml + + + io.swagger + org.apache.pulsar.shade.io.swagger + + + org.apache.bookkeeper + org.apache.pulsar.shade.org.apache.bookkeeper + - - + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index fcf7da8ab293a..c601934505200 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-admin-original Pulsar Client Admin Original - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-admin-api ${project.version} - org.glassfish.jersey.core jersey-client - org.glassfish.jersey.media jersey-media-json-jackson - jakarta.activation jakarta.activation-api - org.glassfish.jersey.media jersey-media-multipart - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - javax.xml.bind jaxb-api @@ -91,30 +79,25 @@ javax.activation runtime - com.google.guava guava - com.google.code.gson gson - ${project.groupId} pulsar-package-core ${project.version} - org.hamcrest hamcrest test - @@ -141,7 +124,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index c9ef631edd5d0..8bd4d350031dc 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-all Pulsar Client All - ${project.groupId} @@ -70,7 +67,6 @@ test - @@ -108,7 +104,6 @@ - maven-antrun-plugin @@ -121,15 +116,12 @@ - + - org.apache.maven.plugins @@ -146,11 +138,10 @@ true true false - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec commons-collections:commons-collections @@ -176,8 +167,7 @@ commons-*:* io.swagger:* io.airlift:* - - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:* com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* @@ -205,7 +195,7 @@ org.apache.commons:commons-compress org.tukaani:xz - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -213,7 +203,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -315,24 +305,24 @@ org.apache.pulsar.shade.org.glassfish - io.grpc - org.apache.pulsar.shade.io.grpc + io.grpc + org.apache.pulsar.shade.io.grpc - okio - org.apache.pulsar.shade.okio + okio + org.apache.pulsar.shade.okio - com.squareup - org.apache.pulsar.shade.com.squareup + com.squareup + org.apache.pulsar.shade.com.squareup - io.opencensus - org.apache.pulsar.shade.io.opencensus + io.opencensus + org.apache.pulsar.shade.io.opencensus - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty org.objenesis @@ -389,14 +379,13 @@ - - + + - - - 4.0.0 - - - org.apache.pulsar - pulsar - 3.0.3-SNAPSHOT - .. - - - pulsar-client-api - Pulsar Client :: API - - - - - - com.google.protobuf - protobuf-java - provided - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - + + com.google.protobuf + protobuf-java + provided + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-client-api/src/main/resources/findbugsExclude.xml b/pulsar-client-api/src/main/resources/findbugsExclude.xml index 9d73ac29a7bdb..353f01a7bb285 100644 --- a/pulsar-client-api/src/main/resources/findbugsExclude.xml +++ b/pulsar-client-api/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index af5e5ed57b0f4..aba76d9cc261f 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-auth-athenz jar Athenz authentication plugin for java client - - ${project.groupId} pulsar-client-original ${project.parent.version} true - com.yahoo.athenz athenz-zts-java-client-core - com.yahoo.athenz athenz-cert-refresher - org.bouncycastle bcpkix-jdk18on - com.google.guava guava - org.apache.commons commons-lang3 - @@ -77,7 +67,6 @@ ${pulsar.client.compiler.release} - org.apache.maven.plugins maven-enforcer-plugin @@ -108,7 +97,6 @@ - org.gaul modernizer-maven-plugin @@ -126,7 +114,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -144,7 +131,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml +++ b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-auth-sasl jar SASL authentication plugin for java client - - ${project.groupId} pulsar-client-original ${project.parent.version} true - com.google.guava guava - org.apache.commons commons-lang3 - org.projectlombok lombok - javax.ws.rs javax.ws.rs-api - org.glassfish.jersey.core jersey-client - - @@ -78,7 +67,6 @@ ${pulsar.client.compiler.release} - org.apache.maven.plugins maven-enforcer-plugin @@ -109,7 +97,6 @@ - org.gaul modernizer-maven-plugin @@ -127,7 +114,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index c46e62f68cea9..341e466d8c611 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-messagecrypto-bc jar Message crypto for End to End encryption with BouncyCastleProvider - ${project.groupId} @@ -40,7 +37,6 @@ ${project.parent.version} provided - ${project.groupId} bouncy-castle-bc @@ -48,9 +44,7 @@ pkg true - - diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index 83ab1777dd9f9..004fd715aa84c 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client Pulsar Client Java - ${project.groupId} @@ -49,11 +46,8 @@ ${project.version} - - - org.apache.maven.plugins maven-dependency-plugin @@ -89,7 +83,6 @@ - maven-antrun-plugin @@ -102,15 +95,12 @@ - + - org.apache.maven.plugins @@ -125,10 +115,9 @@ true true false - - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original org.apache.bookkeeper:* org.apache.commons:commons-lang3 commons-codec:commons-codec @@ -154,19 +143,17 @@ commons-*:* io.swagger:* io.airlift:* - - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common com.yahoo.datasketches:sketches-core org.objenesis:* org.yaml:snakeyaml - org.apache.avro:* com.thoughtworks.paranamer:paranamer org.apache.commons:commons-compress org.tukaani:xz - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc com.fasterxml.jackson.core:jackson-annotations @@ -174,7 +161,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -302,14 +289,13 @@ - - + + - com.github.spotbugs spotbugs-maven-plugin @@ -323,7 +309,6 @@ - - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-tools-api Pulsar Client Tools API Pulsar Client Tools API - ${project.groupId} @@ -66,7 +64,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -82,5 +79,4 @@ - diff --git a/pulsar-client-tools-customcommand-example/pom.xml b/pulsar-client-tools-customcommand-example/pom.xml index 387f5707d4a24..407b5436fdc73 100644 --- a/pulsar-client-tools-customcommand-example/pom.xml +++ b/pulsar-client-tools-customcommand-example/pom.xml @@ -1,3 +1,4 @@ + - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. 4.0.0 @@ -31,7 +32,7 @@ Pulsar CLI Custom command example - org.apache.pulsar + io.streamnative pulsar-client-tools-api ${project.version} provided diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml index 1b5020c944d2e..90dcb5da39002 100644 --- a/pulsar-client-tools-test/pom.xml +++ b/pulsar-client-tools-test/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-tools-test Pulsar Client Tools Test Pulsar Client Tools Test - ${project.groupId} @@ -89,7 +87,6 @@ - org.apache.maven.plugins maven-deploy-plugin @@ -126,8 +123,8 @@ copy filters - - + + diff --git a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml +++ b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-tools Pulsar Client Tools Pulsar Client Tools - com.beust @@ -108,7 +106,6 @@ io.swagger swagger-core - ${project.groupId} @@ -116,9 +113,8 @@ ${project.version} test - - org.apache.pulsar + io.streamnative pulsar-io-batch-discovery-triggerers ${project.version} test @@ -136,9 +132,7 @@ org.apache.commons commons-text - - @@ -158,7 +152,6 @@ - @@ -178,7 +171,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -194,5 +186,4 @@ - diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index a1237dad17298..72bbe1d389218 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-client-original Pulsar Client Java - ${project.groupId} pulsar-client-api ${project.parent.version} - ${project.groupId} pulsar-common ${project.parent.version} - ${project.groupId} bouncy-castle-bc ${project.parent.version} pkg - ${project.groupId} @@ -59,7 +53,6 @@ ${project.parent.version} true - io.netty netty-codec-http @@ -72,7 +65,6 @@ io.netty netty-codec-socks - io.netty netty-resolver-dns @@ -87,50 +79,40 @@ netty-resolver-dns-native-macos osx-x86_64 - org.apache.commons commons-lang3 - org.asynchttpclient async-http-client - com.typesafe.netty netty-reactive-streams - org.slf4j slf4j-api - commons-codec commons-codec - com.yahoo.datasketches sketches-core - com.google.code.gson gson - - org.apache.avro avro ${avro.version} - org.apache.avro avro-protobuf @@ -142,36 +124,30 @@ - joda-time joda-time provided - com.google.protobuf protobuf-java provided - com.fasterxml.jackson.module jackson-module-jsonSchema - net.jcip jcip-annotations - com.github.spotbugs spotbugs-annotations provided true - ${project.groupId} @@ -179,22 +155,18 @@ ${project.parent.version} test - org.skyscreamer jsonassert ${skyscreamer.version} test - org.awaitility awaitility test - - @@ -228,7 +200,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -242,7 +213,6 @@ - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml index 92ec9e934ee1e..44758c90c75de 100644 --- a/pulsar-client/src/main/resources/findbugsExclude.xml +++ b/pulsar-client/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 260a862c5263b..fad0b8a72df56 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-common Pulsar Common Common libraries needed by client/broker/tools - ${project.groupId} pulsar-client-api ${project.version} - ${project.groupId} pulsar-client-admin-api ${project.version} - io.swagger swagger-annotations - org.slf4j slf4j-api - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.module jackson-module-parameter-names - com.fasterxml.jackson.datatype jackson-datatype-jsr310 - com.fasterxml.jackson.datatype jackson-datatype-jdk8 - com.google.guava guava - io.netty netty-handler - io.netty netty-resolver-dns - io.netty netty-transport-native-epoll linux-x86_64 - io.netty netty-transport-native-unix-common linux-x86_64 - org.apache.bookkeeper bookkeeper-common-allocator @@ -114,17 +97,14 @@ - org.apache.bookkeeper cpu-affinity - io.airlift aircompressor - org.apache.bookkeeper circe-checksum @@ -140,66 +120,54 @@ - io.netty netty-tcnative-boringssl-static - io.netty.incubator netty-incubator-transport-classes-io_uring - io.netty.incubator netty-incubator-transport-native-io_uring linux-x86_64 - io.netty.incubator netty-incubator-transport-native-io_uring linux-aarch_64 - io.netty netty-codec-haproxy - org.apache.commons commons-lang3 - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - javax.ws.rs javax.ws.rs-api - commons-io commons-io - com.github.spotbugs spotbugs-annotations provided true - com.google.protobuf - protobuf-java + protobuf-java - org.bouncycastle @@ -207,20 +175,17 @@ ${bouncycastle.bc-fips.version} test - org.lz4 lz4-java 1.5.0 test - com.github.luben zstd-jni test - org.xerial.snappy snappy-java @@ -230,19 +195,16 @@ com.google.code.gson gson - com.beust jcommander - org.awaitility awaitility test - @@ -252,7 +214,6 @@ ${pulsar.client.compiler.release} - org.gaul modernizer-maven-plugin @@ -270,7 +231,6 @@ - com.github.splunk.lightproto lightproto-maven-plugin @@ -283,7 +243,6 @@ - pl.project13.maven git-commit-id-plugin @@ -308,7 +267,6 @@ - org.codehaus.mojo templating-maven-plugin diff --git a/pulsar-common/src/main/resources/findbugsExclude.xml b/pulsar-common/src/main/resources/findbugsExclude.xml index df161c4b621a7..8216948c12408 100644 --- a/pulsar-common/src/main/resources/findbugsExclude.xml +++ b/pulsar-common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index f0033e0c16c3c..3f5e181f7fd0e 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar - pulsar - 3.0.3-SNAPSHOT - .. - - - pulsar-config-validation - Annotation based config validation for Pulsar - - - - org.slf4j - slf4j-api - - - - - - - org.gaul - modernizer-maven-plugin - - true - 17 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - \ No newline at end of file + + 4.0.0 + + io.streamnative + pulsar + 3.0.0-SNAPSHOT + .. + + pulsar-config-validation + Annotation based config validation for Pulsar + + + org.slf4j + slf4j-api + + + + + + org.gaul + modernizer-maven-plugin + + true + 17 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 62253060291f2..ef133487a1d3b 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-functions-api Pulsar Functions :: API - - org.slf4j slf4j-api - net.jodah typetools @@ -48,16 +44,13 @@ ${project.version} compile - ${project.groupId} pulsar-client-admin-api ${project.version} compile - - @@ -81,16 +74,15 @@ org.apache.maven.plugins maven-checkstyle-plugin - - checkstyle - verify - - check - - + + checkstyle + verify + + check + + - diff --git a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml index 9638cfcca8da9..b74af4c8ceb2a 100644 --- a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-functions-instance Pulsar Functions :: Instance - - org.apache.logging.log4j @@ -46,55 +42,46 @@ org.apache.logging.log4j log4j-core - ${project.groupId} pulsar-functions-utils ${project.version} - ${project.groupId} pulsar-metadata ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-functions-secrets ${project.version} - - + ${project.groupId} pulsar-client-admin-original ${project.version} - - + ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-messagecrypto-bc ${project.parent.version} - org.apache.bookkeeper stream-storage-java-client @@ -109,24 +96,20 @@ - io.grpc grpc-stub - io.grpc grpc-all - io.perfmark perfmark-api runtime - org.apache.bookkeeper bookkeeper-common @@ -141,61 +124,50 @@ - com.yahoo.datasketches sketches-core - com.google.guava guava - com.beust jcommander - net.jodah typetools - io.prometheus simpleclient ${prometheus.version} - io.prometheus simpleclient_hotspot ${prometheus.version} - io.prometheus simpleclient_httpserver ${prometheus.version} - io.prometheus.jmx collector ${prometheus-jmx.version} - org.awaitility awaitility test - - @@ -215,7 +187,6 @@ - @@ -271,5 +242,4 @@ - diff --git a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml index 7fe247d2ab20a..4c04f27f66fbe 100644 --- a/pulsar-functions/instance/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/instance/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-functions-api-examples-builtin Pulsar Functions :: API Examples (NAR) - ${project.groupId} @@ -37,14 +35,36 @@ ${project.version} - org.apache.nifi nifi-nar-maven-plugin + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 52bcd621ec9ff..e9aec9fd86cc6 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-functions-api-examples Pulsar Functions :: API Examples - org.slf4j slf4j-api - ${project.groupId} pulsar-functions-api @@ -52,7 +49,6 @@ ${project.version} - @@ -87,5 +83,4 @@ - diff --git a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index 902ed7fbdf965..ae64fc2bdf294 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.0.3-SNAPSHOT - .. - - - pulsar-functions-local-runner - Pulsar Functions :: Local Runner Shaded - - - - ${project.groupId} - pulsar-functions-local-runner-original - ${project.parent.version} - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - org.asynchttpclient - async-http-client - ${asynchttpclient.version} - jar - true - org/asynchttpclient/config/ahc-default.properties - ${project.build.directory}/classes - - - - - - - - - maven-antrun-plugin - - - shade-ahc-properties - prepare-package - - run - - - - - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - false - - - - - - - org.apache.pulsar:* - org.apache.bookkeeper:* - commons-*:* - org.apache.commons:* - com.fasterxml.jackson.*:* - io.netty:* - com.google.*:* - javax.servlet:* - org.reactivestreams:reactive-streams - org.apache.commons:* - io.swagger:* - org.yaml:snakeyaml - io.perfmark:* - io.prometheus:* - io.prometheus.jmx:* - javax.ws.rs:* - org.tukaani:xz - com.github.zafarkhaja:java-semver - net.java.dev.jna:* - org.apache.zookeeper:* - com.thoughtworks.paranamer:paranamer - jline:* - org.rocksdb:* - org.eclipse.jetty*:* - org.apache.avro:avro - com.beust:* - net.jodah:* - io.airlift:* - com.yahoo.datasketches:* - io.netty.resolver:* - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + ${shadePluginPhase} + + shade + + + false + + + + + + + io.streamnative:* + org.apache.bookkeeper:* + commons-*:* + org.apache.commons:* + com.fasterxml.jackson.*:* + io.netty:* + com.google.*:* + javax.servlet:* + org.reactivestreams:reactive-streams + org.apache.commons:* + io.swagger:* + org.yaml:snakeyaml + io.perfmark:* + io.prometheus:* + io.prometheus.jmx:* + javax.ws.rs:* + org.tukaani:xz + com.github.zafarkhaja:java-semver + net.java.dev.jna:* + org.apache.zookeeper:* + com.thoughtworks.paranamer:paranamer + jline:* + org.rocksdb:* + org.eclipse.jetty*:* + org.apache.avro:avro + com.beust:* + net.jodah:* + io.airlift:* + com.yahoo.datasketches:* + io.netty.resolver:* + + + + + io.streamnative:pulsar-client-original + + ** + + + + org/bouncycastle/** + + + + + + - - com.google - org.apache.pulsar.functions.runtime.shaded.com.google - - - org.apache.jute - org.apache.pulsar.functions.runtime.shaded.org.apache.jute - - - javax.servlet - org.apache.pulsar.functions.runtime.shaded.javax.servlet - - - net.jodah - org.apache.pulsar.functions.runtime.shaded.net.jodah - - - org.lz4 - org.apache.pulsar.functions.runtime.shaded.org.lz4 - - - org.reactivestreams - org.apache.pulsar.functions.runtime.shaded.org.reactivestreams - - - org.apache.commons - org.apache.pulsar.functions.runtime.shaded.org.apache.commons - - - io.swagger - org.apache.pulsar.functions.runtime.shaded.io.swagger - - - org.yaml - org.apache.pulsar.functions.runtime.shaded.org.yaml - - - org.jctools - org.apache.pulsar.functions.runtime.shaded.org.jctools - - - com.squareup.okhttp - org.apache.pulsar.functions.runtime.shaded.com.squareup.okhttp - - - io.grpc - org.apache.pulsar.functions.runtime.shaded.io.grpc - - - io.perfmark - org.apache.pulsar.functions.runtime.shaded.io.perfmark - - - org.joda - org.apache.pulsar.functions.runtime.shaded.org.joda - - - javax.ws.rs - org.apache.pulsar.functions.runtime.shaded.javax.ws.rs - - - io.kubernetes - org.apache.pulsar.functions.runtime.shaded.io.kubernetes - - - io.opencensus - org.apache.pulsar.functions.runtime.shaded.io.opencensus - - - net.jpountz - org.apache.pulsar.functions.runtime.shaded.net.jpountz - - - commons-configuration - org.apache.pulsar.functions.runtime.shaded.commons-configuration - - - org.tukaani - org.apache.pulsar.functions.runtime.shaded.org.tukaani - - - com.github - org.apache.pulsar.functions.runtime.shaded.com.github - - - commons-io - org.apache.pulsar.functions.runtime.shaded.commons-io - - - org.apache.distributedlog - org.apache.pulsar.functions.runtime.shaded.org.apache.distributedlog - - - - com.fasterxml - org.apache.pulsar.functions.runtime.shaded.com.fasterxml - - - org.inferred - org.apache.pulsar.functions.runtime.shaded.org.inferred - - - org.apache.bookkeeper - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - - - org.bookkeeper - org.apache.pulsar.functions.runtime.shaded.org.bookkeeper - - - dlshade - org.apache.pulsar.functions.runtime.shaded.dlshade - - - - org.codehaus.jackson - org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson - - - net.java.dev.jna - org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna - - - org.apache.curator - org.apache.pulsar.functions.runtime.shaded.org.apache.curator - - - javax.validation - org.apache.pulsar.functions.runtime.shaded.javax.validation - - - javax.activation - org.apache.pulsar.functions.runtime.shaded.javax.activation - - - io.prometheus - org.apache.pulsar.functions.runtime.shaded.io.prometheus - - - io.prometheus.jmx - org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx - - - org.apache.zookeeper - org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper - - - io.jsonwebtoken - org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken - - - commons-codec - org.apache.pulsar.functions.runtime.shaded.commons-codec - - - com.thoughtworks.paranamer - org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer - - - org.codehaus.mojo - org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo - - - com.github.luben - org.apache.pulsar.functions.runtime.shaded.com.github.luben - - - jline - org.apache.pulsar.functions.runtime.shaded.jline - - - org.xerial.snappy - org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy - - - javax.annotation - org.apache.pulsar.functions.runtime.shaded.javax.annotation - - - org.checkerframework - org.apache.pulsar.functions.runtime.shaded.org.checkerframework - - - org.apache.yetus - org.apache.pulsar.functions.runtime.shaded.org.apache.yetus - - - commons-cli - org.apache.pulsar.functions.runtime.shaded.commons-cli - - - commons-lang - org.apache.pulsar.functions.runtime.shaded.commons-lang - - - com.squareup.okio - org.apache.pulsar.functions.runtime.shaded.com.squareup.okio - - - org.rocksdb - org.apache.pulsar.functions.runtime.shaded.org.rocksdb - - - org.objenesis - org.apache.pulsar.functions.runtime.shaded.org.objenesis - - - org.eclipse.jetty - org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty - - - org.apache.avro - org.apache.pulsar.functions.runtime.shaded.org.apache.avro - - - avro.shaded - org.apache.pulsar.functions.runtime.shaded.avo.shaded - - - com.yahoo.datasketches - org.apache.pulsar.shaded.com.yahoo.datasketches - - - com.yahoo.sketches - org.apache.pulsar.shaded.com.yahoo.sketches - - - com.beust - org.apache.pulsar.functions.runtime.shaded.com.beust - - - - io.netty - org.apache.pulsar.functions.runtime.shaded.io.netty - - - org.hamcrest - org.apache.pulsar.functions.runtime.shaded.org.hamcrest - - - aj.org - org.apache.pulsar.functions.runtime.shaded.aj.org - - - com.scurrilous - org.apache.pulsar.functions.runtime.shaded.com.scurrilous - - - okio - org.apache.pulsar.functions.runtime.shaded.okio - - + + com.fasterxml + org.apache.pulsar.functions.runtime.shaded.com.fasterxml + + + org.inferred + org.apache.pulsar.functions.runtime.shaded.org.inferred + + + org.apache.bookkeeper + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + + + org.bookkeeper + org.apache.pulsar.functions.runtime.shaded.org.bookkeeper + + + dlshade + org.apache.pulsar.functions.runtime.shaded.dlshade + + + + org.codehaus.jackson + org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson + + + net.java.dev.jna + org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna + + + org.apache.curator + org.apache.pulsar.functions.runtime.shaded.org.apache.curator + + + javax.validation + org.apache.pulsar.functions.runtime.shaded.javax.validation + + + javax.activation + org.apache.pulsar.functions.runtime.shaded.javax.activation + + + io.prometheus + org.apache.pulsar.functions.runtime.shaded.io.prometheus + + + io.prometheus.jmx + org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx + + + org.apache.zookeeper + org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper + + + io.jsonwebtoken + org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken + + + commons-codec + org.apache.pulsar.functions.runtime.shaded.commons-codec + + + com.thoughtworks.paranamer + org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer + + + org.codehaus.mojo + org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo + + + com.github.luben + org.apache.pulsar.functions.runtime.shaded.com.github.luben + + + jline + org.apache.pulsar.functions.runtime.shaded.jline + + + org.xerial.snappy + org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy + + + javax.annotation + org.apache.pulsar.functions.runtime.shaded.javax.annotation + + + org.checkerframework + org.apache.pulsar.functions.runtime.shaded.org.checkerframework + + + org.apache.yetus + org.apache.pulsar.functions.runtime.shaded.org.apache.yetus + + + commons-cli + org.apache.pulsar.functions.runtime.shaded.commons-cli + + + commons-lang + org.apache.pulsar.functions.runtime.shaded.commons-lang + + + com.squareup.okio + org.apache.pulsar.functions.runtime.shaded.com.squareup.okio + + + org.rocksdb + org.apache.pulsar.functions.runtime.shaded.org.rocksdb + + + org.objenesis + org.apache.pulsar.functions.runtime.shaded.org.objenesis + + + org.eclipse.jetty + org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty + + + org.apache.avro + org.apache.pulsar.functions.runtime.shaded.org.apache.avro + + + avro.shaded + org.apache.pulsar.functions.runtime.shaded.avo.shaded + + + com.yahoo.datasketches + org.apache.pulsar.shaded.com.yahoo.datasketches + + + com.yahoo.sketches + org.apache.pulsar.shaded.com.yahoo.sketches + + + com.beust + org.apache.pulsar.functions.runtime.shaded.com.beust + + + + io.netty + org.apache.pulsar.functions.runtime.shaded.io.netty + + + org.hamcrest + org.apache.pulsar.functions.runtime.shaded.org.hamcrest + + + aj.org + org.apache.pulsar.functions.runtime.shaded.aj.org + + + com.scurrilous + org.apache.pulsar.functions.runtime.shaded.com.scurrilous + + + okio + org.apache.pulsar.functions.runtime.shaded.okio + + - - org.asynchttpclient - org.apache.pulsar.functions.runtime.shaded.org.asynchttpclient - - - io.airlift - org.apache.pulsar.functions.runtime.shaded.io.airlift - - - - - - - - - + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index 4a87563584e71..611ab9ecc4dd5 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.0.3-SNAPSHOT - .. - - - pulsar-functions-local-runner-original - Pulsar Functions :: Local Runner original - - - - ${project.groupId} - pulsar-functions-runtime - ${project.parent.version} - - - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - - io.grpc - grpc-all - - - - io.perfmark - perfmark-api - runtime - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - check - verify - - check - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar-functions + 3.0.0-SNAPSHOT + .. + + pulsar-functions-local-runner-original + Pulsar Functions :: Local Runner original + + + ${project.groupId} + pulsar-functions-runtime + ${project.parent.version} + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + io.grpc + grpc-all + + + io.perfmark + perfmark-api + runtime + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + check + verify + + check + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml index ad183eb5d9427..c7c41fe9ea252 100644 --- a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index ef91b1a8de3d9..5b8ecb76ab9b5 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-functions Pulsar Functions :: Parent - main @@ -41,24 +39,23 @@ !true - - proto - api-java - java-examples - java-examples-builtin - utils - instance - runtime - runtime-all - worker - secrets - localrun - localrun-shaded - + + proto + api-java + java-examples + java-examples-builtin + utils + instance + runtime + runtime-all + worker + secrets + localrun + localrun-shaded + - - - core-modules + + core-modules proto api-java @@ -72,8 +69,6 @@ secrets localrun - - + - diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 339282cf6cc17..75ad2eb4fdd3f 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 3.0.3-SNAPSHOT - - - pulsar-functions-proto - Pulsar Functions :: Proto - - - - - com.google.protobuf - protobuf-java - - - - com.google.protobuf - protobuf-java-util - - - - javax.annotation - javax.annotation-api - - - - io.grpc - grpc-all - - - io.netty - * - - - - - - io.perfmark - perfmark-api - runtime - - - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - true - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - - - - generate-sources - - compile - compile-custom - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-functions + 3.0.0-SNAPSHOT + + pulsar-functions-proto + Pulsar Functions :: Proto + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + javax.annotation + javax.annotation-api + + + io.grpc + grpc-all + + + io.netty + * + + + + + io.perfmark + perfmark-api + runtime + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} + true + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} + + + + generate-sources + + compile + compile-custom + + + + + + diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index de4a899c1e219..e3ede4d5429d8 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - - pulsar-functions-runtime-all Pulsar Functions :: Runtime All - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-client-api ${project.version} - org.apache.avro avro ${avro.version} - com.fasterxml.jackson.core jackson-databind - com.google.protobuf @@ -95,30 +85,24 @@ gson ${gson.version} - - org.slf4j slf4j-api - org.apache.logging.log4j log4j-slf4j-impl - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml index 190d9be92940b..84e6ce44f51e4 100644 --- a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-instance - 30 - - - pulsar.log.appender - RollingFile - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - RollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}*log.gz - - - 30d - - - - - - BkRollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}.bk*log.gz - - - 30d - - - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - BkRollingFile - - - - info - - ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-instance + 30 + + + pulsar.log.appender + RollingFile + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + RollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}*log.gz + + + 30d + + + + + + BkRollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}.bk*log.gz + + + 30d + + + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + BkRollingFile + + + + info + + ${sys:pulsar.log.appender} + ${sys:pulsar.log.level} + + + + diff --git a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml index f86d03e41793f..fe478be5b8667 100644 --- a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-kubernetes-instance - 30 - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - Console - - - - info - - Console - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-kubernetes-instance + 30 + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + Console + + + + info + + Console + ${sys:pulsar.log.level} + + + + diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 4a19e9d3e4fb6..df389d4268336 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-functions-runtime Pulsar Functions :: Runtime - - ${project.groupId} pulsar-functions-instance ${project.version} - ${project.groupId} pulsar-functions-secrets ${project.version} - com.beust jcommander - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.core jackson-databind - io.kubernetes client-java @@ -84,9 +75,7 @@ pulsar-broker-common ${project.version} - - @@ -106,7 +95,6 @@ - org.apache.maven.plugins @@ -154,5 +142,4 @@ - diff --git a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml index b4a4e5926de3f..976abe7c54bb2 100644 --- a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 57beb0c4701ed..3001025a5951e 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-functions-secrets Pulsar Functions :: Secrets - io.kubernetes @@ -50,7 +48,6 @@ - ${project.groupId} pulsar-functions-proto @@ -76,7 +73,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -109,5 +105,4 @@ - diff --git a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml index 0cab8ae6a418a..7d4464ae1c08d 100644 --- a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-functions-utils Pulsar Functions :: Utils - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-proto ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-config-validation ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.core jackson-annotations - com.fasterxml.jackson.core jackson-databind - com.google.code.gson gson - net.jodah typetools - net.bytebuddy byte-buddy - org.zeroturnaround zt-zip 1.17 - - ${project.groupId} - pulsar-client-original - ${project.version} + ${project.groupId} + pulsar-client-original + ${project.version} - ${project.groupId} pulsar-package-core ${project.version} - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - @@ -137,7 +119,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -153,5 +134,4 @@ - diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 4cddaf1cc5be7..d033ce5e628b6 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-functions-worker Pulsar Functions :: Worker - - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-functions-runtime ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-admin-original ${project.version} - org.glassfish.jersey.media jersey-media-json-jackson - jakarta.activation jakarta.activation-api - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.media jersey-media-multipart - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.apache.distributedlog distributedlog-core @@ -118,17 +100,14 @@ - org.apache.bookkeeper bookkeeper-server - commons-io commons-io - io.swagger swagger-core @@ -143,14 +122,11 @@ - io.prometheus simpleclient_jetty - - ${project.groupId} pulsar-io-cassandra @@ -158,7 +134,6 @@ pom test - ${project.groupId} pulsar-io-twitter @@ -166,7 +141,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples @@ -174,7 +148,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples-builtin @@ -182,9 +155,7 @@ pom test - - @@ -204,7 +175,6 @@ - maven-dependency-plugin @@ -257,7 +227,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -273,7 +242,6 @@ - org.apache.maven.plugins @@ -321,5 +289,4 @@ - diff --git a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index ae88bfd771db0..d29689b52418c 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-aerospike Pulsar IO :: Aerospike - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.aerospike aerospike-client-bc @@ -63,9 +57,7 @@ org.bouncycastle bcpkix-jdk18on - - @@ -91,6 +83,4 @@ - - diff --git a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/alluxio/pom.xml b/pulsar-io/alluxio/pom.xml index fcd6cf46797bf..f0ddbf427f684 100644 --- a/pulsar-io/alluxio/pom.xml +++ b/pulsar-io/alluxio/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.0.3-SNAPSHOT - - - - 2.7.3 - 4.1.11 - 1.37.0 - - - pulsar-io-alluxio - Pulsar IO :: Alluxio - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - test - - - - org.alluxio - alluxio-core-client-fs - ${alluxio.version} - - - grpc-netty - io.grpc - - - - - - org.alluxio - alluxio-minicluster - ${alluxio.version} - test - - - org.glassfish - javax.el - - - grpc-netty - io.grpc - - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.google.guava - guava - - - - - io.grpc - grpc-netty - ${grpc.version} - - - - io.dropwizard.metrics - metrics-jvm - ${metrics.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + pulsar-io + io.streamnative + 3.0.0-SNAPSHOT + + + 2.7.3 + 4.1.11 + 1.37.0 + + pulsar-io-alluxio + Pulsar IO :: Alluxio + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + test + + + org.alluxio + alluxio-core-client-fs + ${alluxio.version} + + + grpc-netty + io.grpc + + + + + org.alluxio + alluxio-minicluster + ${alluxio.version} + test + + + org.glassfish + javax.el + + + grpc-netty + io.grpc + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.google.guava + guava + + + + io.grpc + grpc-netty + ${grpc.version} + + + io.dropwizard.metrics + metrics-jvm + ${metrics.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 5e34531feb4e2..5d236be733ce0 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-aws Pulsar IO :: IO AWS - ${project.groupId} pulsar-io-core ${project.version} - com.google.code.gson gson - com.amazonaws aws-java-sdk-sts - software.amazon.awssdk sts 2.10.56 - - + - diff --git a/pulsar-io/aws/src/main/resources/findbugsExclude.xml b/pulsar-io/aws/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/aws/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/aws/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index fcacca3c25fb0..f62ef085aa17d 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.0.3-SNAPSHOT - - - pulsar-io-batch-data-generator - Pulsar IO :: Batch Data Generator - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-io-batch-discovery-triggerers - ${project.version} - - - - org.springframework - spring-context - ${spring.version} - - - - io.codearte.jfairy - jfairy - 0.5.9 - - - - org.apache.avro - avro - ${avro.version} - - - - ${project.groupId} - pulsar-functions-local-runner-original - ${project.version} - test - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.0.0-SNAPSHOT + + pulsar-io-batch-data-generator + Pulsar IO :: Batch Data Generator + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-io-batch-discovery-triggerers + ${project.version} + + + org.springframework + spring-context + ${spring.version} + + + io.codearte.jfairy + jfairy + 0.5.9 + + + org.apache.avro + avro + ${avro.version} + + + ${project.groupId} + pulsar-functions-local-runner-original + ${project.version} + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml index 0b1652ef62881..1d0d64fba7e67 100644 --- a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-batch-discovery-triggerers Pulsar IO :: Batch Discovery Triggerers - ${project.groupId} pulsar-io-core ${project.version} - com.cronutils cron-utils ${cron-utils.version} - org.springframework spring-context ${spring.version} - - @@ -73,5 +66,4 @@ - diff --git a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - org.apache.pulsar - pulsar-io - 3.0.3-SNAPSHOT - - 4.0.0 - - pulsar-io-canal - Pulsar IO :: Canal - - - 1.1.5 - - - - - ${project.groupId} - pulsar-io-common - ${project.version} - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.alibaba - fastjson - 1.2.83 - - - - org.springframework - spring-core - ${spring.version} - - - org.springframework - spring-aop - ${spring.version} - - - org.springframework - spring-context - ${spring.version} - - - org.springframework - spring-jdbc - ${spring.version} - - - org.springframework - spring-orm - ${spring.version} - - - - com.alibaba.otter - canal.protocol - ${canal.version} - - - ch.qos.logback - * - - - - - com.alibaba.otter - canal.client - ${canal.version} - - - ch.qos.logback - * - - - org.springframework - * - - - - - - org.apache.logging.log4j - log4j-core - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - - \ No newline at end of file + + + io.streamnative + pulsar-io + 3.0.0-SNAPSHOT + + 4.0.0 + pulsar-io-canal + Pulsar IO :: Canal + + 1.1.5 + + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.alibaba + fastjson + 1.2.83 + + + org.springframework + spring-core + ${spring.version} + + + org.springframework + spring-aop + ${spring.version} + + + org.springframework + spring-context + ${spring.version} + + + org.springframework + spring-jdbc + ${spring.version} + + + org.springframework + spring-orm + ${spring.version} + + + com.alibaba.otter + canal.protocol + ${canal.version} + + + ch.qos.logback + * + + + + + com.alibaba.otter + canal.client + ${canal.version} + + + ch.qos.logback + * + + + org.springframework + * + + + + + org.apache.logging.log4j + log4j-core + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index a05a79a1234e9..1ca3a66ab37d2 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-cassandra Pulsar IO :: Cassandra - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.datastax.cassandra cassandra-driver-core - - diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 91e6e53316f3c..122527bb4bbb6 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-io - 3.0.3-SNAPSHOT - - - pulsar-io-common - Pulsar IO :: IO Common - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.0.0-SNAPSHOT + + pulsar-io-common + Pulsar IO :: IO Common + + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/common/src/main/resources/findbugsExclude.xml b/pulsar-io/common/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/common/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-core Pulsar IO :: IO - ${project.groupId} @@ -37,7 +35,6 @@ ${project.version} - @@ -59,5 +56,4 @@ - diff --git a/pulsar-io/core/src/main/resources/findbugsExclude.xml b/pulsar-io/core/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/core/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/core/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.0.3-SNAPSHOT - - - pulsar-io-data-generator - Pulsar IO :: Data Generator - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-config-validation - ${project.version} - - - - io.codearte.jfairy - jfairy - 0.5.9 - - - - org.apache.avro - avro - ${avro.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.0.0-SNAPSHOT + + pulsar-io-data-generator + Pulsar IO :: Data Generator + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-config-validation + ${project.version} + + + io.codearte.jfairy + jfairy + 0.5.9 + + + org.apache.avro + avro + ${avro.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index b358ef2e9f0f9..f9790674c81f5 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-debezium-core Pulsar IO :: Debezium :: Core - - ${project.groupId} pulsar-io-core ${project.version} provided - io.debezium debezium-core ${debezium.version} - org.apache.commons commons-lang3 - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-io-kafka-connect-adaptor ${project.version} - org.apache.kafka connect-runtime @@ -77,21 +69,18 @@ - ${project.groupId} pulsar-broker ${project.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-broker @@ -99,21 +88,17 @@ test test-jar - io.debezium debezium-connector-mysql ${debezium.version} test - ${project.groupId} pulsar-client-original ${project.version} test - - diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 7b0920ceefbe3..81455d0000bd8 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io-debezium - 3.0.3-SNAPSHOT - - - pulsar-io-debezium-mongodb - Pulsar IO :: Debezium :: mongodb - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - provided - - - - ${project.groupId} - pulsar-io-debezium-core - ${project.version} - - - - io.debezium - debezium-connector-mongodb - ${debezium.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - + + 4.0.0 + + io.streamnative + pulsar-io-debezium + 3.0.0-SNAPSHOT + + pulsar-io-debezium-mongodb + Pulsar IO :: Debezium :: mongodb + + + ${project.groupId} + pulsar-io-core + ${project.version} + provided + + + ${project.groupId} + pulsar-io-debezium-core + ${project.version} + + + io.debezium + debezium-connector-mongodb + ${debezium.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 26b582472700a..3c562c9677c94 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-debezium-mssql Pulsar IO :: Debezium :: Microsoft SQL - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-sqlserver ${debezium.version} - - @@ -61,5 +54,4 @@ - diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index 27d510702e1b1..472614768da07 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-debezium-mysql Pulsar IO :: Debezium :: mysql - ${project.groupId} @@ -37,21 +35,17 @@ ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-mysql ${debezium.version} - - @@ -61,8 +55,6 @@ - - @@ -71,5 +63,4 @@ - diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 80754791ae934..3b735ebac4964 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-debezium-oracle Pulsar IO :: Debezium :: oracle - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-oracle ${debezium.version} - - @@ -61,5 +54,4 @@ - diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 3f2ee701b120b..976fc07c68162 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-debezium Pulsar IO :: Debezium - @@ -70,7 +68,6 @@ - core mysql @@ -79,5 +76,4 @@ oracle mssql - diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index 2934af7c415a7..a9f3957a26b08 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-debezium-postgres Pulsar IO :: Debezium :: postgres - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-postgres ${debezium.version} - org.postgresql postgresql ${debezium.postgresql.version} runtime - - @@ -68,5 +60,4 @@ - diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 9b1a8aebd4fa0..603d9d8b6dd8d 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-docs Pulsar IO :: Docs - - ${project.groupId} pulsar-io-core @@ -45,7 +42,6 @@ com.beust jcommander - ${project.groupId} @@ -218,7 +214,6 @@ ${project.version} - @@ -267,5 +262,4 @@ - diff --git a/pulsar-io/docs/src/main/resources/findbugsExclude.xml b/pulsar-io/docs/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/docs/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/docs/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-dynamodb Pulsar IO :: DynamoDB - - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-instance ${project.version} provided - ${project.groupId} pulsar-io-aws ${project.version} - org.apache.commons commons-lang3 - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.dataformat jackson-dataformat-cbor - com.google.code.gson gson - com.amazonaws aws-java-sdk-core - com.amazonaws @@ -96,9 +83,7 @@ 1.5.1 - - @@ -107,5 +92,4 @@ - diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 5726e17a9e948..9dbd0b4078d36 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.0.3-SNAPSHOT - - - pulsar-io-flume - Pulsar IO :: Flume - - - 1.8.2 - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.commons - commons-collections4 - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - org.apache.flume - flume-ng-node - 1.9.0 - pom - - - org.apache.avro - avro-ipc - - - avro - org.apache.avro - - - commons-collections - commons-collections - - - - - org.apache.avro - avro - ${avro.version} - - - commons-lang - commons-lang - 2.6 - - - org.apache.avro - avro-ipc - ${avro.version} - - - org.mortbay.jetty - servlet-api - - - io.netty - netty - - - commons-collections - commons-collections - - - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.curator - curator-test - ${curator.version} - test - - - - io.dropwizard.metrics - metrics-core - test - - - - org.xerial.snappy - snappy-java - test - - - - com.google.guava - guava - ${guava.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + + com.google.guava + guava + ${guava.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index 1c2f7d0d63dc1..dfd65c0b0b71c 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.0.3-SNAPSHOT - - pulsar-io-hbase - Pulsar IO :: Hbase - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.google.guava - guava - - - - org.apache.hbase - hbase-client - ${hbase.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hbase - hbase-common - ${hbase.version} - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml index c2b520a765643..460cbd497c15f 100644 --- a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml +++ b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml @@ -1,3 +1,4 @@ + - - hbase.cluster.distributed - true - - - hbase.rootdir - hdfs://localhost:8020/hbase - - - hbase.zookeeper.quorum - localhost - - - hbase.zookeeper.property.clientPort - 2181 - - - zookeeper.znode.parent - /hbase - + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://localhost:8020/hbase + + + hbase.zookeeper.quorum + localhost + + + hbase.zookeeper.property.clientPort + 2181 + + + zookeeper.znode.parent + /hbase + diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 758b65246d908..56d5e2fc6c195 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -1,3 +1,4 @@ + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - - \ No newline at end of file + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + + diff --git a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml index 980349c4d8d5e..93756f7f28461 100644 --- a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - fs.defaultFS - hdfs://0.0.0.0:8020 - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec, + + fs.defaultFS + hdfs://0.0.0.0:8020 + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec, org.apache.hadoop.io.compress.DefaultCodec, org.apache.hadoop.io.compress.SnappyCodec - + diff --git a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml index bb722f1f63470..1b35ebd585012 100644 --- a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml +++ b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml @@ -1,3 +1,4 @@ + - - dfs.replication - 1 - - - dfs.client.use.datanode.hostname - true - - - dfs.support.append - true - + + dfs.replication + 1 + + + dfs.client.use.datanode.hostname + true + + + dfs.support.append + true + diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index cd12b11a8764e..5104424dbdafc 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -1,3 +1,4 @@ + - - fs.defaultFS - hdfs://0.0.0.0:8020 - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec, + + fs.defaultFS + hdfs://0.0.0.0:8020 + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec, org.apache.hadoop.io.compress.DefaultCodec, org.apache.hadoop.io.compress.SnappyCodec - + diff --git a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml index bb722f1f63470..1b35ebd585012 100644 --- a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml +++ b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml @@ -1,3 +1,4 @@ + - - dfs.replication - 1 - - - dfs.client.use.datanode.hostname - true - - - dfs.support.append - true - + + dfs.replication + 1 + + + dfs.client.use.datanode.hostname + true + + + dfs.support.append + true + diff --git a/pulsar-io/http/pom.xml b/pulsar-io/http/pom.xml index 76399ee6348e4..9525885f26461 100644 --- a/pulsar-io/http/pom.xml +++ b/pulsar-io/http/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.0.3-SNAPSHOT - - - pulsar-io-http - Pulsar IO :: HTTP - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - - - - org.apache.avro - avro - ${avro.version} - - - - org.apache.pulsar - pulsar-client-original - ${project.version} - test - - - - com.github.tomakehurst - wiremock-jre8 - ${wiremock.version} - test - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.0.0-SNAPSHOT + + pulsar-io-http + Pulsar IO :: HTTP + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + org.apache.avro + avro + ${avro.version} + + + io.streamnative + pulsar-client-original + ${project.version} + test + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index ec3d1c8d2f727..b436d7b988881 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.0.3-SNAPSHOT - - - pulsar-io-influxdb - Pulsar IO :: InfluxDB - - - - ${project.groupId} - pulsar-io-common - ${project.version} - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - com.influxdb - influxdb-client-java - 4.0.0 - - - - org.influxdb - influxdb-java - 2.22 - - - com.squareup.okhttp3 - * - - - - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.0.0-SNAPSHOT + + pulsar-io-influxdb + Pulsar IO :: InfluxDB + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + com.influxdb + influxdb-client-java + 4.0.0 + + + org.influxdb + influxdb-java + 2.22 + + + com.squareup.okhttp3 + * + + + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index 0a6e91bb7d85a..cbf39587e9cb6 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 3.0.3-SNAPSHOT + io.streamnative + 3.0.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-clickhouse Pulsar IO :: Jdbc :: ClickHouse - ${project.groupId} @@ -45,13 +42,12 @@ all - * - * + * + * - diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 7870eaa64545e..034b51fedd8f9 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 3.0.3-SNAPSHOT + io.streamnative + 3.0.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-core Pulsar IO :: Jdbc :: Core - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - org.apache.avro avro ${avro.version} - com.fasterxml.jackson.core jackson-databind - com.google.guava guava - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.protobuf protobuf-java provided - - - \ No newline at end of file + diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index 41ab7a9f66a66..737fdd0704c32 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 3.0.3-SNAPSHOT + io.streamnative + 3.0.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-mariadb Pulsar IO :: Jdbc :: Mariadb - ${project.groupId} @@ -52,4 +49,4 @@ - \ No newline at end of file + diff --git a/pulsar-io/jdbc/openmldb/pom.xml b/pulsar-io/jdbc/openmldb/pom.xml index 48a5f478c0c23..09d36b016f732 100644 --- a/pulsar-io/jdbc/openmldb/pom.xml +++ b/pulsar-io/jdbc/openmldb/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 3.0.3-SNAPSHOT + io.streamnative + 3.0.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-openmldb Pulsar IO :: Jdbc :: OpenMLDB - ${project.groupId} @@ -60,7 +57,6 @@ runtime - diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index 11f9118e51454..0e2b8d229867f 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom @@ -31,12 +31,10 @@ openmldb - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-jdbc Pulsar IO :: Jdbc - diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index b22f30aaa1838..a6ae052e42a07 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 3.0.3-SNAPSHOT + io.streamnative + 3.0.0-SNAPSHOT 4.0.0 - pulsar-io-jdbc-postgres Pulsar IO :: Jdbc :: Postgres - ${project.groupId} @@ -44,7 +41,6 @@ runtime - diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index b7faa5eaf5f88..f920de0adcc67 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 3.0.3-SNAPSHOT + io.streamnative + 3.0.0-SNAPSHOT 4.0.0 pulsar-io-jdbc-sqlite Pulsar IO :: Jdbc :: Sqlite - ${project.groupId} @@ -60,7 +58,6 @@ test - @@ -69,4 +66,4 @@ - \ No newline at end of file + diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index b8a13d239759e..5b9f8cbdc99a0 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-kafka-connect-adaptor-nar Pulsar IO :: Kafka Connect Adaptor NAR - ${project.groupId} @@ -37,8 +35,6 @@ ${project.version} - - @@ -48,7 +44,30 @@ pulsar-io-kafka-connect-adaptor-${project.version} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index e820b17568fba..35cd8318836e8 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-kafka-connect-adaptor Pulsar IO :: Kafka Connect Adaptor - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-common ${project.version} compile - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - org.apache.kafka connect-runtime @@ -79,7 +72,6 @@ - org.apache.kafka connect-json @@ -91,7 +83,6 @@ - org.apache.kafka connect-api @@ -103,7 +94,6 @@ - ${project.groupId} @@ -116,17 +106,14 @@ - io.netty netty-buffer - org.apache.commons commons-lang3 - io.confluent @@ -139,14 +126,12 @@ - ${project.groupId} pulsar-broker ${project.version} test - org.apache.kafka connect-file @@ -159,14 +144,12 @@ - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-broker @@ -174,38 +157,37 @@ test test-jar - org.bouncycastle bc-fips ${bouncycastle.bc-fips.version} test - org.apache.avro avro ${avro.version} provided - com.google.protobuf protobuf-java provided - org.asynchttpclient async-http-client test - com.typesafe.netty netty-reactive-streams test + + com.google.protobuf + protobuf-java + 3.5.1 + - diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 290e03c7b340e..70aff1a6fce77 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-kafka Pulsar IO :: Kafka - @@ -44,42 +42,35 @@ - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-client-original ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.guava guava - org.apache.kafka kafka-clients @@ -91,43 +82,35 @@ - io.confluent kafka-schema-registry-client ${confluent.version} - io.confluent kafka-avro-serializer ${confluent.version} - io.jsonwebtoken jjwt-impl - io.jsonwebtoken jjwt-jackson - org.hamcrest hamcrest test - org.awaitility awaitility test - - diff --git a/pulsar-io/kafka/src/main/resources/findbugsExclude.xml b/pulsar-io/kafka/src/main/resources/findbugsExclude.xml index 3fc8f67c8640e..a48959506a69a 100644 --- a/pulsar-io/kafka/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/kafka/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-kinesis Pulsar IO :: Kinesis - 2.2.8 0.14.0 @@ -37,116 +35,96 @@ 1.9.0 2.3.0 - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-io-aws ${project.version} - - org.apache.pulsar + io.streamnative pulsar-functions-instance ${project.version} test - org.apache.commons commons-lang3 - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.dataformat jackson-dataformat-cbor - org.apache.avro avro ${avro.version} - com.github.wnameless.json json-flattener ${json-flattener.version} - com.google.code.gson gson - com.amazonaws aws-java-sdk-core - software.amazon.kinesis amazon-kinesis-client ${amazon-kinesis-client.version} - com.amazonaws amazon-kinesis-producer ${amazon-kinesis-producer.version} - - + com.google.flatbuffers flatbuffers-java ${flatbuffers-java.version} - javax.xml.bind jaxb-api ${jaxb-api.version} - org.testcontainers localstack test - org.awaitility awaitility test - - @@ -155,5 +133,4 @@ - diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 25c6c59d7a189..c0bfce38fc3c3 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar - pulsar-io - 3.0.3-SNAPSHOT + io.streamnative + pulsar-io + 3.0.0-SNAPSHOT - pulsar-io-mongo Pulsar IO :: MongoDB - 4.1.2 - ${project.groupId} @@ -69,7 +64,6 @@ guava - diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index 49d31267d8528..8b9d89fe739ba 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 3.0.3-SNAPSHOT - - - pulsar-io-netty - Pulsar IO :: Netty - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - io.netty - netty-handler - - - - io.netty - netty-codec-http - - - - com.google.guava - guava - - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 3.0.0-SNAPSHOT + + pulsar-io-netty + Pulsar IO :: Netty + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + io.netty + netty-handler + + + io.netty + netty-codec-http + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/netty/src/main/resources/findbugsExclude.xml b/pulsar-io/netty/src/main/resources/findbugsExclude.xml index 3a52062fc9233..6b8906a775bd3 100644 --- a/pulsar-io/netty/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/netty/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index 2186d262826c1..fccec986b8569 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-nsq Pulsar IO :: NSQ - ${project.groupId} pulsar-io-core ${project.version} - com.sproutsocial nsq-j ${nsq-client.version} - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - ${project.groupId} pulsar-io-common ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - - - - org.apache.nifi - nifi-nar-maven-plugin - - + + + org.apache.nifi + nifi-nar-maven-plugin + + diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 5b543dd77b1a8..0d39a02d508d8 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io Pulsar IO :: Parent - main @@ -77,7 +75,6 @@ alluxio - pulsar-io-tests @@ -110,7 +107,6 @@ alluxio - pulsar-io-elastic-tests @@ -119,7 +115,6 @@ elastic-search - pulsar-io-kafka-connect-tests @@ -131,7 +126,6 @@ debezium - core-modules @@ -145,7 +139,6 @@ - @@ -163,5 +156,4 @@ - diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index e35b5516858ff..43888fee0dc4f 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-rabbitmq Pulsar IO :: RabbitMQ - - ${project.groupId} pulsar-io-common @@ -58,28 +55,23 @@ pulsar-client-original ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.guava guava - com.rabbitmq amqp-client ${rabbitmq-client.version} - org.apache.qpid qpid-broker @@ -91,9 +83,7 @@ awaitility test - - @@ -102,5 +92,4 @@ - diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index ef66a3f37c6d1..69daa0aed3f28 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.0.3-SNAPSHOT - - - pulsar-io-redis - Pulsar IO :: Redis - - - - ${project.groupId} - pulsar-io-common - ${project.version} - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - io.lettuce - lettuce-core - 5.0.2.RELEASE - - - com.google.guava - guava - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - com.github.kstyrc - embedded-redis - 0.6 - test - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.0.0-SNAPSHOT + + pulsar-io-redis + Pulsar IO :: Redis + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + io.lettuce + lettuce-core + 5.0.2.RELEASE + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + com.github.kstyrc + embedded-redis + 0.6 + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 9d2f7410a92fd..6f68e52bf21d3 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 3.0.3-SNAPSHOT - - - - 8.11.3 - - - pulsar-io-solr - Pulsar IO :: Solr - - - - ${project.groupId} - pulsar-io-common - ${project.version} - - - ${project.parent.groupId} - pulsar-io-core - ${project.parent.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - org.apache.solr - solr-solrj - ${solr.version} - - - org.apache.solr - solr-core - ${solr.version} - - - jose4j - org.bitbucket.b_c - - - test - - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-collections4 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 3.0.0-SNAPSHOT + + + 8.11.3 + + pulsar-io-solr + Pulsar IO :: Solr + + + ${project.groupId} + pulsar-io-common + ${project.version} + + + ${project.parent.groupId} + pulsar-io-core + ${project.parent.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + org.apache.solr + solr-solrj + ${solr.version} + + + org.apache.solr + solr-core + ${solr.version} + + + jose4j + org.bitbucket.b_c + + + test + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-collections4 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + + diff --git a/pulsar-io/solr/src/main/resources/findbugsExclude.xml b/pulsar-io/solr/src/main/resources/findbugsExclude.xml index f49ff86c90caf..47c3d73af5f06 100644 --- a/pulsar-io/solr/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/solr/src/main/resources/findbugsExclude.xml @@ -1,22 +1,23 @@ - - - \ No newline at end of file + + + + diff --git a/pulsar-io/solr/src/test/resources/solr.xml b/pulsar-io/solr/src/test/resources/solr.xml index 495a71f72f3f2..9ce4d37f4beef 100644 --- a/pulsar-io/solr/src/test/resources/solr.xml +++ b/pulsar-io/solr/src/test/resources/solr.xml @@ -1,3 +1,4 @@ + - - ${host:} - ${jetty.port:8983} - ${hostContext:solr} - ${genericCoreNodeNames:true} - ${zkClientTimeout:30000} - ${distribUpdateSoTimeout:600000} - ${distribUpdateConnTimeout:60000} - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - - - ${socketTimeout:600000} - ${connTimeout:60000} - - \ No newline at end of file + + ${host:} + ${jetty.port:8983} + ${hostContext:solr} + ${genericCoreNodeNames:true} + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + ${socketTimeout:600000} + ${connTimeout:60000} + + diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 3f8c8bd01ca43..1314bf881e79d 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-io-twitter Pulsar IO :: Twitter - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.twitter hbc-core ${hbc-core.version} - org.apache.commons commons-collections4 - org.apache.commons commons-lang3 - - ${project.groupId} - pulsar-io-common - ${project.version} + ${project.groupId} + pulsar-io-common + ${project.version} - - @@ -80,5 +69,4 @@ - diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 0c6db1bbd8dfc..cc0f22a2db64e 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-metadata Pulsar Metadata - org.apache.pulsar + io.streamnative pulsar-common ${project.version} - org.apache.zookeeper zookeeper @@ -56,13 +53,11 @@ - io.dropwizard.metrics metrics-core test - org.apache.zookeeper zookeeper @@ -83,55 +78,44 @@ - ${project.groupId} testmocks ${project.version} test - org.xerial.snappy snappy-java test - org.awaitility awaitility test - org.apache.bookkeeper bookkeeper-server - io.etcd jetcd-core - - io.etcd jetcd-test test - com.github.ben-manes.caffeine caffeine - io.prometheus simpleclient - - @@ -151,7 +135,6 @@ - org.apache.maven.plugins maven-jar-plugin diff --git a/pulsar-metadata/src/main/resources/findbugsExclude.xml b/pulsar-metadata/src/main/resources/findbugsExclude.xml index 21578b9f8cb5c..d6eb8623eb73e 100644 --- a/pulsar-metadata/src/main/resources/findbugsExclude.xml +++ b/pulsar-metadata/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/pulsar-metadata/src/test/resources/findbugsExclude.xml b/pulsar-metadata/src/test/resources/findbugsExclude.xml index e40b2a0fd9a12..bf075c39791cf 100644 --- a/pulsar-metadata/src/test/resources/findbugsExclude.xml +++ b/pulsar-metadata/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - \ No newline at end of file + + + + + + + + + diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index ceea81e074e92..17ee17d52c773 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar-package-management - org.apache.pulsar - 3.0.3-SNAPSHOT - - 4.0.0 - - pulsar-package-bookkeeper-storage - Apache Pulsar :: Package Management :: BookKeeper Storage - - - - ${project.groupId} - pulsar-package-core - ${project.version} - - - - org.apache.distributedlog - distributedlog-core - - - net.jpountz.lz4 - lz4 - - - - - - org.apache.zookeeper - zookeeper - tests - test - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - io.netty - netty-tcnative - - - - - - org.hamcrest - hamcrest - test - - - - - io.dropwizard.metrics - metrics-core - test - - - - - org.xerial.snappy - snappy-java - test - - - - ${project.groupId} - managed-ledger - ${project.version} - test - - - - ${project.groupId} - testmocks - ${project.version} - - - + + + pulsar-package-management + io.streamnative + 3.0.0-SNAPSHOT + + 4.0.0 + pulsar-package-bookkeeper-storage + Apache Pulsar :: Package Management :: BookKeeper Storage + + + ${project.groupId} + pulsar-package-core + ${project.version} + + + org.apache.distributedlog + distributedlog-core + + + net.jpountz.lz4 + lz4 + + + + + org.apache.zookeeper + zookeeper + tests + test + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + io.netty + netty-tcnative + + + + + org.hamcrest + hamcrest + test + + + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + + ${project.groupId} + managed-ledger + ${project.version} + test + + + ${project.groupId} + testmocks + ${project.version} + + diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index 59095f293a6c3..f307b22a42f3d 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar-package-management - org.apache.pulsar - 3.0.3-SNAPSHOT - - 4.0.0 - - pulsar-package-core - Apache Pulsar :: Package Management :: Core - - - - ${project.groupId} - pulsar-client-admin-api - ${project.parent.version} - - - - com.google.guava - guava - - - - org.apache.commons - commons-lang3 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - ${pulsar.client.compiler.release} - - - - org.gaul - modernizer-maven-plugin - - true - 17 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - + + + pulsar-package-management + io.streamnative + 3.0.0-SNAPSHOT + + 4.0.0 + pulsar-package-core + Apache Pulsar :: Package Management :: Core + + + ${project.groupId} + pulsar-client-admin-api + ${project.parent.version} + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + ${pulsar.client.compiler.release} + + + + org.gaul + modernizer-maven-plugin + + true + 17 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + diff --git a/pulsar-package-management/core/src/main/resources/findbugsExclude.xml b/pulsar-package-management/core/src/main/resources/findbugsExclude.xml index 3a2e998dce984..bd8d2c4f77efa 100644 --- a/pulsar-package-management/core/src/main/resources/findbugsExclude.xml +++ b/pulsar-package-management/core/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - pulsar-package-management - org.apache.pulsar - 3.0.3-SNAPSHOT - - 4.0.0 - - pulsar-package-filesystem-storage - Apache Pulsar :: Package Management :: Filesystem Storage - - - - ${project.groupId} - pulsar-package-core - ${project.parent.version} - - - - com.google.guava - guava - - - - ${project.groupId} - testmocks - ${project.parent.version} - test - - - + + + pulsar-package-management + io.streamnative + 3.0.0-SNAPSHOT + + 4.0.0 + pulsar-package-filesystem-storage + Apache Pulsar :: Package Management :: Filesystem Storage + + + ${project.groupId} + pulsar-package-core + ${project.parent.version} + + + com.google.guava + guava + + + ${project.groupId} + testmocks + ${project.parent.version} + test + + diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index db5a8e186347d..a2f02a5d2c1e8 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar - org.apache.pulsar - 3.0.3-SNAPSHOT - .. - - 4.0.0 - - pulsar-package-management - pom - Apache Pulsar :: Package Management - - core - bookkeeper-storage - filesystem-storage - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - spotbugs - verify - - check - - - - - - + + + pulsar + io.streamnative + 3.0.0-SNAPSHOT + .. + + 4.0.0 + pulsar-package-management + pom + Apache Pulsar :: Package Management + + core + bookkeeper-storage + filesystem-storage + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 9b3b1beb72213..6cef823ccaa82 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-proxy Pulsar Proxy - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-websocket ${project.version} - org.apache.commons commons-lang3 - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.eclipse.jetty jetty-proxy - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.media jersey-media-json-jackson - com.google.guava guava - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - javax.xml.bind jaxb-api - com.sun.activation javax.activation - io.prometheus simpleclient - io.prometheus simpleclient_hotspot - io.prometheus simpleclient_servlet - io.prometheus simpleclient_jetty - ${project.groupId} pulsar-broker ${project.version} test - ${project.groupId} testmocks ${project.version} test - org.awaitility awaitility test - com.beust jcommander - org.apache.logging.log4j log4j-core - com.github.seancfoley ipaddress @@ -283,5 +254,4 @@ - diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index b51b1b450917b..5e589018b7e8e 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - pom - - org.apache.pulsar - pulsar - 3.0.3-SNAPSHOT - - - pulsar-sql - Pulsar SQL :: Parent - - - - 3.14.9 - - 1.17.2 - 213 - - - - - - - com.squareup.okhttp3 - okhttp - ${okhttp3.version} - - - com.squareup.okhttp3 - okhttp-urlconnection - ${okhttp3.version} - - - com.squareup.okhttp3 - logging-interceptor - ${okhttp3.version} - - - com.squareup.okio - okio - ${okio.version} - - - - - org.jline - jline-reader - ${jline3.version} - - - org.jline - jline-terminal - ${jline3.version} - - - org.jline - jline-terminal-jna - ${jline3.version} - - - - - org.slf4j - log4j-over-slf4j - ${slf4j.version} - - - org.slf4j - slf4j-jdk14 - ${slf4j.version} - - - - io.airlift - bom - ${airlift.version} - pom - import - - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - - - - main - - - disableSqlMainProfile - - !true - - - - presto-pulsar - presto-pulsar-plugin - presto-distribution - - - - pulsar-sql-tests - - presto-pulsar - presto-pulsar-plugin - presto-distribution - - - + 3.14.9 + + 1.17.2 + 213 + + + + + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + com.squareup.okhttp3 + okhttp-urlconnection + ${okhttp3.version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + + + com.squareup.okio + okio + ${okio.version} + + + + org.jline + jline-reader + ${jline3.version} + + + org.jline + jline-terminal + ${jline3.version} + + + org.jline + jline-terminal-jna + ${jline3.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-jdk14 + ${slf4j.version} + + + io.airlift + bom + ${airlift.version} + pom + import + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + + + main + + + disableSqlMainProfile + + !true + + + + presto-pulsar + presto-pulsar-plugin + presto-distribution + + + + pulsar-sql-tests + + presto-pulsar + presto-pulsar-plugin + presto-distribution + + + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index 94e6a5f4a5c8a..dbd8ec200f36c 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-sql - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-presto-distribution Pulsar SQL :: Pulsar Presto Distribution - false 2.34 @@ -42,7 +39,6 @@ 2.5.1 4.0.1 - org.glassfish.jersey.core @@ -74,7 +70,6 @@ jersey-client ${jersey.version} - io.trino trino-server-main @@ -103,13 +98,11 @@ - io.trino trino-cli ${trino.version} - io.airlift launcher @@ -117,7 +110,6 @@ tar.gz bin - io.airlift launcher @@ -125,13 +117,11 @@ tar.gz properties - org.objenesis objenesis ${objenesis.version} - com.twitter.common objectsize @@ -143,56 +133,44 @@ - - com.fasterxml.jackson.core jackson-core - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.core jackson-annotations - com.fasterxml.jackson.datatype jackson-datatype-joda - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.datatype jackson-datatype-guava - com.fasterxml.jackson.datatype jackson-datatype-jdk8 - com.fasterxml.jackson.datatype jackson-datatype-jsr310 - com.fasterxml.jackson.dataformat jackson-dataformat-smile - - @@ -205,7 +183,6 @@ netty 3.10.6.Final - org.apache.maven maven-core @@ -295,7 +272,6 @@ - @@ -305,7 +281,6 @@ ${skipBuildDistribution} - org.apache.maven.plugins maven-assembly-plugin @@ -329,7 +304,6 @@ - com.mycila license-maven-plugin @@ -354,7 +328,6 @@ - skipBuildDistributionDisabled diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml index 96c0421c71515..8439d60ba339e 100644 --- a/pulsar-sql/presto-distribution/src/assembly/assembly.xml +++ b/pulsar-sql/presto-distribution/src/assembly/assembly.xml @@ -1,3 +1,4 @@ + - - bin - - tar.gz - dir - - false - - - ${basedir}/LICENSE - LICENSE - . - 644 - - - ${basedir}/src/main/resources/launcher.properties - launcher.properties - bin/ - 644 - - - - - ${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/ - plugin/ - - - ${basedir}/src/main/resources/conf/ - conf/ - - - - - lib/ - true - runtime - - io.airlift:launcher:tar.gz:bin:${airlift.version} - io.airlift:launcher:tar.gz:properties:${airlift.version} - *:tar.gz - - - - - - io.airlift:launcher:tar.gz:bin:${airlift.version} - - true - 755 - - - - - io.airlift:launcher:tar.gz:properties:${airlift.version} - - true - - - \ No newline at end of file + + bin + + tar.gz + dir + + false + + + ${basedir}/LICENSE + LICENSE + . + 644 + + + ${basedir}/src/main/resources/launcher.properties + launcher.properties + bin/ + 644 + + + + + ${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/ + plugin/ + + + ${basedir}/src/main/resources/conf/ + conf/ + + + + + lib/ + true + runtime + + io.airlift:launcher:tar.gz:bin:${airlift.version} + io.airlift:launcher:tar.gz:properties:${airlift.version} + *:tar.gz + + + + + + io.airlift:launcher:tar.gz:bin:${airlift.version} + + true + 755 + + + + + io.airlift:launcher:tar.gz:properties:${airlift.version} + + true + + + diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index 189cbaa34d27b..8a35373ebfb3a 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 3.0.3-SNAPSHOT - - - pulsar-presto-connector - Pulsar SQL :: Pulsar Presto Connector - - - - - ${project.groupId} - pulsar-presto-connector-original - ${project.version} - - - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.3.0 - - false - true - posix - - src/assembly/assembly.xml - - - - - package - package - - single - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-sql + 3.0.0-SNAPSHOT + + pulsar-presto-connector + Pulsar SQL :: Pulsar Presto Connector + + + ${project.groupId} + pulsar-presto-connector-original + ${project.version} + + + ${project.groupId} + bouncy-castle-bc + ${project.version} + pkg + true + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + false + true + posix + + src/assembly/assembly.xml + + + + + package + package + + single + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + diff --git a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml b/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml index 6650abfda3fc3..649baf7318003 100644 --- a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml +++ b/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml @@ -1,3 +1,4 @@ + - - bin - - tar.gz - dir - - - - / - false - runtime - - jakarta.ws.rs:jakarta.ws.rs-api - - - - \ No newline at end of file + + bin + + tar.gz + dir + + + + / + false + runtime + + jakarta.ws.rs:jakarta.ws.rs-api + + + + diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index e5337dc4683f4..edabcbb839bb0 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 3.0.3-SNAPSHOT - - - pulsar-presto-connector-original - Pulsar SQL - Pulsar Presto Connector - Pulsar SQL :: Pulsar Presto Connector Packaging - - - 2.1.2 - 1.8.4 - - - - - io.airlift - bootstrap - - - org.apache.logging.log4j - log4j-to-slf4j - - - - - io.airlift - json - - - - org.apache.avro - avro - ${avro.version} - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - ${project.groupId} - managed-ledger - ${project.version} - - - - org.jctools - jctools-core - ${jctools.version} - - - - com.dslplatform - dsl-json - ${dslJson.verson} - - - - io.trino - trino-plugin-toolkit - ${trino.version} - - - - - io.trino - trino-spi - ${trino.version} - provided - - - - joda-time - joda-time - ${joda.version} - - - - io.trino - trino-record-decoder - ${trino.version} - - - - javax.annotation - javax.annotation-api - ${javax.annotation-api.version} - - - - io.jsonwebtoken - jjwt-impl - ${jsonwebtoken.version} - test - - - - io.trino - trino-main - ${trino.version} - test - - - - io.trino - trino-testing - ${trino.version} - test - - - - org.apache.pulsar + + 4.0.0 + + io.streamnative + pulsar-sql + 3.0.0-SNAPSHOT + + pulsar-presto-connector-original + Pulsar SQL - Pulsar Presto Connector + Pulsar SQL :: Pulsar Presto Connector Packaging + + 2.1.2 + 1.8.4 + + + + io.airlift + bootstrap + + + org.apache.logging.log4j + log4j-to-slf4j + + + + + io.airlift + json + + + org.apache.avro + avro + ${avro.version} + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + ${project.groupId} + managed-ledger + ${project.version} + + + org.jctools + jctools-core + ${jctools.version} + + + com.dslplatform + dsl-json + ${dslJson.verson} + + + io.trino + trino-plugin-toolkit + ${trino.version} + + + + io.trino + trino-spi + ${trino.version} + provided + + + joda-time + joda-time + ${joda.version} + + + io.trino + trino-record-decoder + ${trino.version} + + + javax.annotation + javax.annotation-api + ${javax.annotation-api.version} + + + io.jsonwebtoken + jjwt-impl + ${jsonwebtoken.version} + test + + + io.trino + trino-main + ${trino.version} + test + + + io.trino + trino-testing + ${trino.version} + test + + + io.streamnative + pulsar-broker + ${project.version} + test + + + io.streamnative + testmocks + ${project.version} + test + + + org.eclipse.jetty + jetty-http + ${jetty.version} + test + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + ${shadePluginPhase} + + shade + + + true + true + + + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original + io.streamnative:managed-ledger + io.streamnative:pulsar-metadata + org.glassfish.jersey*:* + javax.ws.rs:* + javax.annotation:* + org.glassfish.hk2*:* + org.eclipse.jetty:* + + + + + io.streamnative:pulsar-client-original + + ** + + + + org/bouncycastle/** + + + + + + org.glassfish + org.apache.pulsar.shade.org.glassfish + + + javax.ws + org.apache.pulsar.shade.javax.ws + + + javax.annotation + org.apache.pulsar.shade.javax.annotation + + + jersey + org.apache.pulsar.shade.jersey + + + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty + + + + + + + + + + + + + + + + test-jar-dependencies + + + maven.test.skip + !true + + + + + ${project.groupId} pulsar-broker ${project.version} + test-jar test - - - org.apache.pulsar - testmocks - ${project.version} - test - - - - org.eclipse.jetty - jetty-http - ${jetty.version} - test - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - true - true - - - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original - org.apache.pulsar:managed-ledger - org.apache.pulsar:pulsar-metadata - - org.glassfish.jersey*:* - javax.ws.rs:* - javax.annotation:* - org.glassfish.hk2*:* - - org.eclipse.jetty:* - - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - org.glassfish - org.apache.pulsar.shade.org.glassfish - - - javax.ws - org.apache.pulsar.shade.javax.ws - - - javax.annotation - org.apache.pulsar.shade.javax.annotation - - - jersey - org.apache.pulsar.shade.jersey - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - - - - - - - - - - - - - - - - test-jar-dependencies - - - maven.test.skip - !true - - - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - - + + + diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 5fabafb6782f5..d5d84c2ce91f4 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar - pulsar - 3.0.3-SNAPSHOT - .. - - - pulsar-testclient - Pulsar Test Client - Pulsar Test Client - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.apache.zookeeper - zookeeper - - - ch.qos.logback - logback-core - - - ch.qos.logback - logback-classic - - - io.netty - netty-tcnative - - - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - ${project.groupId} - pulsar-client-messagecrypto-bc - ${project.version} - true - - - - ${project.groupId} - pulsar-broker - ${project.version} - - - - commons-configuration - commons-configuration - - - - com.beust - jcommander - compile - - - - org.hdrhistogram - HdrHistogram - - - - com.fasterxml.jackson.core - jackson-databind - - - - org.awaitility - awaitility - test - - - - com.github.tomakehurst - wiremock-jre8 - ${wiremock.version} - test - - - - - - - - - test-jar-dependencies - - - maven.test.skip - !true - - - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar + 3.0.0-SNAPSHOT + .. + + pulsar-testclient + Pulsar Test Client + Pulsar Test Client + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.apache.zookeeper + zookeeper + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + io.netty + netty-tcnative + + + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + ${project.groupId} + pulsar-client-messagecrypto-bc + ${project.version} + true + + + ${project.groupId} + pulsar-broker + ${project.version} + + + commons-configuration + commons-configuration + + + com.beust + jcommander + compile + + + org.hdrhistogram + HdrHistogram + + + com.fasterxml.jackson.core + jackson-databind + + + org.awaitility + awaitility + test + + + com.github.tomakehurst + wiremock-jre8 + ${wiremock.version} + test + + + + + + test-jar-dependencies + + + maven.test.skip + !true + + + + + ${project.groupId} + pulsar-broker + ${project.version} + test-jar + test + + + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index d806aa28464cc..1bc0dbccd64f3 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-transaction-parent - 3.0.3-SNAPSHOT - - - pulsar-transaction-common - Pulsar Transaction :: Common - - + + 4.0.0 + + io.streamnative + pulsar-transaction-parent + 3.0.0-SNAPSHOT + + pulsar-transaction-common + Pulsar Transaction :: Common + - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + diff --git a/pulsar-transaction/common/src/main/resources/findbugsExclude.xml b/pulsar-transaction/common/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-transaction/common/src/main/resources/findbugsExclude.xml +++ b/pulsar-transaction/common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-transaction-parent - 3.0.3-SNAPSHOT - - - pulsar-transaction-coordinator - Pulsar Transaction :: Coordinator - - - - - ${project.groupId} - pulsar-common - ${project.version} - - - - ${project.groupId} - managed-ledger - ${project.version} - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.awaitility - awaitility - test - - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - set-system-properties - - - - - proto_path - ${project.parent.parent.basedir} - - - proto_search_strategy - 2 - - - - - - - - com.github.splunk.lightproto - lightproto-maven-plugin - ${lightproto-maven-plugin.version} - - - - generate - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - + + 4.0.0 + + io.streamnative + pulsar-transaction-parent + 3.0.0-SNAPSHOT + + pulsar-transaction-coordinator + Pulsar Transaction :: Coordinator + + + ${project.groupId} + pulsar-common + ${project.version} + + + ${project.groupId} + managed-ledger + ${project.version} + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.awaitility + awaitility + test + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.codehaus.mojo + properties-maven-plugin + + + initialize + + set-system-properties + + + + + proto_path + ${project.parent.parent.basedir} + + + proto_search_strategy + 2 + + + + + + + + com.github.splunk.lightproto + lightproto-maven-plugin + ${lightproto-maven-plugin.version} + + + + generate + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + diff --git a/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml b/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml index a81fce11f4d21..f1cad02b28a88 100644 --- a/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml +++ b/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - pulsar-transaction-parent Pulsar Transaction :: Parent - common coordinator - - - com.github.spotbugs spotbugs-maven-plugin @@ -68,7 +63,5 @@ - - diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 8edbaa1b3927e..c14a0d8356a8f 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - pulsar-websocket Pulsar WebSocket - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} managed-ledger ${project.parent.version} test - org.apache.commons commons-lang3 - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.inject jersey-hk2 - com.google.code.gson gson - io.swagger swagger-core - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - - + org.eclipse.jetty.websocket websocket-api ${jetty.version} - - + org.eclipse.jetty.websocket websocket-server ${jetty.version} - + org.eclipse.jetty.websocket javax-websocket-client-impl @@ -115,9 +101,7 @@ org.hdrhistogram HdrHistogram - - @@ -137,7 +121,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -155,7 +138,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-websocket/src/main/resources/findbugsExclude.xml b/pulsar-websocket/src/main/resources/findbugsExclude.xml index b7f6b0bf31d51..26c9af667da61 100644 --- a/pulsar-websocket/src/main/resources/findbugsExclude.xml +++ b/pulsar-websocket/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + @@ -199,4 +199,4 @@ - \ No newline at end of file + diff --git a/src/assembly-source-package.xml b/src/assembly-source-package.xml index 00f00bfe3a5c7..bae52588edcf6 100644 --- a/src/assembly-source-package.xml +++ b/src/assembly-source-package.xml @@ -31,7 +31,7 @@ . - + true @@ -39,14 +39,11 @@ src/*.py docker/pulsar/scripts/*.sh docker/pulsar/scripts/*.py - data/** logs/** - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}.*] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?maven-eclipse\.xml] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.project] @@ -72,12 +68,10 @@ %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.externalToolBuilders(/.*)?] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.deployables(/.*)?] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.wtpmodules(/.*)?] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?cobertura\.ser] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.versionsBackup] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?dependency-reduced-pom\.xml] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.releaseBackup] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties] @@ -86,7 +80,7 @@ ${project.build.directory}/maven-shared-archive-resources/META-INF - + src diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh index 3a6d266345f30..9c6a4d3223a70 100755 --- a/src/check-binary-license.sh +++ b/src/check-binary-license.sh @@ -61,7 +61,7 @@ EXIT=0 # Check all bundled jars are mentioned in LICENSE for J in $JARS; do - echo $J | grep -q "org.apache.pulsar" + echo $J | grep -q "io.streamnative" if [ $? == 0 ]; then continue fi @@ -104,7 +104,7 @@ if [ "$NO_PRESTO" -ne 1 ]; then for J in $JARS; do - echo $J | grep -q "org.apache.pulsar" + echo $J | grep -q "io.streamnative" if [ $? == 0 ]; then continue fi diff --git a/src/findbugs-exclude.xml b/src/findbugs-exclude.xml index 8f289b83a7be1..de4d3f7a93371 100644 --- a/src/findbugs-exclude.xml +++ b/src/findbugs-exclude.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/src/idea-code-style.xml b/src/idea-code-style.xml index 810dee34a3494..b49c523c322b2 100644 --- a/src/idea-code-style.xml +++ b/src/idea-code-style.xml @@ -1,3 +1,4 @@ + - - - - - \ No newline at end of file + diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 345be8f4d2c06..151752c98f7a1 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -1,4 +1,4 @@ - + - - + + file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-28164 - file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-29425 - file name: zookeeper-3.8.0.jar - ]]> + e395c1d8a71557b7569cc6a83487b2e30e2e58fe CVE-2021-34429 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-28164 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-29425 - file name: zookeeper-prometheus-metrics-3.8.0.jar - ]]> + 849e8ece2845cb0185d721233906d487a7f1e4cf CVE-2021-34429 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-28164 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-29425 - file name: zookeeper-jute-3.8.0.jar - ]]> + 6560f966bcf1aa78d27bcfa75fb6c4463a72c6c5 CVE-2021-34429 - - file name: debezium-connector-postgres-1.7.2.Final.jar - ]]> + 69c1edfa7d89531af511fcd07e8516fa450f746a CVE-2021-23214 - - - + - file name: mariadb-java-client-2.7.5.jar - ]]> + 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 CVE-2022-27376 CVE-2022-27377 @@ -162,43 +159,36 @@ CVE-2022-27386 CVE-2022-27387 - - file name: google-http-client-gson-1.41.0.jar - ]]> + 1a754a5dd672218a2ac667d7ff2b28df7a5a240e CVE-2022-25647 - commons-net is not used at all and therefore commons-net vulnerability CVE-2021-37533 is a false positive. CVE-2021-37533 - fredsmith utils library is not used at all. CVE-2021-4277 is a false positive. CVE-2021-4277 - It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-25194 is a false positive. CVE-2023-25194 - It treat pulsar-io-kafka-connect-adaptor as a lib of Kafka, CVE-2021-34917 is a false positive. CVE-2022-34917 - yaml_project is not used at all. Any CVEs reported for yaml_project are false positives. cpe:/a:yaml_project:yaml - flat_project is not used at all. cpe:/a:flat_project:flat - \ No newline at end of file + diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index 1a82702105bfb..e000f056b5e1f 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -1,4 +1,4 @@ - + - - - Ignore netty CVEs in GRPC shaded Netty. - .*grpc-netty-shaded.* - cpe:/a:netty:netty - - - Suppress all pulsar-presto-distribution vulnerabilities - .*pulsar-presto-distribution-.* - .* - - - Suppress libthrift-0.12.0.jar vulnerabilities - org.apache.thrift:libthrift:0.12.0 - .* - - - - - + + Ignore netty CVEs in GRPC shaded Netty. + .*grpc-netty-shaded.* + cpe:/a:netty:netty + + + Suppress all pulsar-presto-distribution vulnerabilities + .*pulsar-presto-distribution-.* + .* + + + Suppress libthrift-0.12.0.jar vulnerabilities + org.apache.thrift:libthrift:0.12.0 + .* + + + + file name: msgpack-core-0.9.0.jar - ]]> - 87d9ce0b22de48428fa32bb8ad476e18b6969548 - CVE-2022-41719 - - - - - + 87d9ce0b22de48428fa32bb8ad476e18b6969548 + CVE-2022-41719 + + + + file name: elasticsearch-java-8.1.0.jar CVE-2022-23712 is only related to Elastic server. - ]]> - edf5be04cbc2eafc51540ba33f9536e788b9d60b - CVE-2022-23712 - - - + edf5be04cbc2eafc51540ba33f9536e788b9d60b + CVE-2022-23712 + + + file name: elasticsearch-rest-client-8.1.0.jar CVE-2022-23712 is only related to Elastic server. - ]]> - 10e7aa09f10955a074c0a574cb699344d2745df1 - CVE-2022-23712 - - - - - + 10e7aa09f10955a074c0a574cb699344d2745df1 + CVE-2022-23712 + + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:22.0) - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2018-10237 - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2018-10237 + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:22.0) - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2020-8908 - - - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2020-8908 + + + + file name: avro-1.10.2.jar - ]]> - a65aaa91c1aeceb3dd4859dbb9765d1c2063f5a2 - CVE-2021-43045 - - - + a65aaa91c1aeceb3dd4859dbb9765d1c2063f5a2 + CVE-2021-43045 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14668 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14668 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14669 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14669 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14670 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14670 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14671 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14671 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14672 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14672 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-15024 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-15024 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-16535 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-16535 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-18657 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-18657 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2021-25263 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2021-25263 + + + file name: logback-core-1.1.3.jar - ]]> - e3c02049f2dbbc764681b40094ecf0dcbc99b157 - cpe:/a:qos:logback - - - + e3c02049f2dbbc764681b40094ecf0dcbc99b157 + cpe:/a:qos:logback + + + file name: rocketmq-acl-4.5.2.jar - ]]> - 0e2bd9c162280cd79c2ea0f67f174ee5d7b84ddd - cpe:/a:apache:rocketmq - - - - ^pkg:maven/org\.springframework/spring.*$ - CVE-2016-1000027 - - - + 0e2bd9c162280cd79c2ea0f67f174ee5d7b84ddd + cpe:/a:apache:rocketmq + + + Ignored since we are not vulnerable + ^pkg:maven/org\.springframework/spring.*$ + CVE-2016-1000027 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - cpe:/a:qos:logback - - - + d90276fff414f06cb375f2057f6778cd63c6082f + cpe:/a:qos:logback + + + file name: logback-core-1.1.3.jar - ]]> - e3c02049f2dbbc764681b40094ecf0dcbc99b157 - CVE-2017-5929 - - - + e3c02049f2dbbc764681b40094ecf0dcbc99b157 + CVE-2017-5929 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - CVE-2017-5929 - - - + d90276fff414f06cb375f2057f6778cd63c6082f + CVE-2017-5929 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - CVE-2021-42550 - - - - Ignore etdc CVEs in jetcd - .*jetcd.* - cpe:/a:etcd:etcd - - - Ignore etdc CVEs in jetcd - .*jetcd.* - cpe:/a:redhat:etcd - - - Ignore grpc CVEs in jetcd - .*jetcd-grpc.* - cpe:/a:grpc:grpc - - - - - + d90276fff414f06cb375f2057f6778cd63c6082f + CVE-2021-42550 + + + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:etcd:etcd + + + Ignore etdc CVEs in jetcd + .*jetcd.* + cpe:/a:redhat:etcd + + + Ignore grpc CVEs in jetcd + .*jetcd-grpc.* + cpe:/a:grpc:grpc + + + + file name: bc-fips-1.0.2.jar - ]]> - 4fb5db5f03d00f6a94e43b78d097978190e4abb2 - CVE-2020-26939 - - - + 4fb5db5f03d00f6a94e43b78d097978190e4abb2 + CVE-2020-26939 + + + file name: bcpkix-fips-1.0.2.jar - ]]> - 543bc7a08cdba0172e95e536b5f7ca61f021253d - CVE-2020-15522 - - - + 543bc7a08cdba0172e95e536b5f7ca61f021253d + CVE-2020-15522 + + + file name: bcpkix-fips-1.0.2.jar - ]]> - 543bc7a08cdba0172e95e536b5f7ca61f021253d - CVE-2020-26939 - - - - - + 543bc7a08cdba0172e95e536b5f7ca61f021253d + CVE-2020-26939 + + + + file name: openstack-swift-2.5.0.jar - ]]> - d99d0eab2e01d69d8a326fc152427fbd759af88a - CVE-2016-0738 - - - + d99d0eab2e01d69d8a326fc152427fbd759af88a + CVE-2016-0738 + + + file name: openstack-swift-2.5.0.jar - ]]> - d99d0eab2e01d69d8a326fc152427fbd759af88a - CVE-2017-16613 - - - + d99d0eab2e01d69d8a326fc152427fbd759af88a + CVE-2017-16613 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2018-14432 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2018-14432 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2018-20170 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2018-20170 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12689 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12689 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12690 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12690 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12691 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12691 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12692 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12692 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2021-3563 - - - - - + file name: org.apache.pulsar:pulsar-io-solr:2.10.0-SNAPSHOT - ]]> - ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ - cpe:/a:apache:pulsar - - - + ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ + cpe:/a:apache:pulsar + + + file name: org.apache.pulsar:pulsar-io-solr:2.10.0-SNAPSHOT - ]]> - ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ - cpe:/a:apache:solr - - - - - + ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ + cpe:/a:apache:solr + + + + file name: debezium-connector-mysql-1.9.7.Final.jar - ]]> - 74c623b4a9b231e2f0e8f6053b38abd3bc487ce2 - CVE-2017-15945 - - - + 74c623b4a9b231e2f0e8f6053b38abd3bc487ce2 + CVE-2017-15945 + + + file name: mysql-binlog-connector-java-0.27.2.jar - ]]> - 23294cd730e29c00b8ddfbde517dfc955aba090f - CVE-2017-15945 - - - + 23294cd730e29c00b8ddfbde517dfc955aba090f + CVE-2017-15945 + + + file name: debezium-connector-postgres-1.9.7.Final.jar - ]]> - 300ff0bbf795643e914b7c8a6d6ba812a8354d62 - CVE-2015-0241 - CVE-2015-0242 - CVE-2015-0243 - CVE-2015-0244 - CVE-2015-3166 - CVE-2015-3167 - CVE-2016-0766 - CVE-2016-0768 - CVE-2016-0773 - CVE-2016-5423 - CVE-2016-5424 - CVE-2016-7048 - CVE-2017-14798 - CVE-2017-7484 - CVE-2018-1115 - CVE-2019-10127 - CVE-2019-10128 - CVE-2019-10210 - CVE-2019-10211 - CVE-2020-25694 - CVE-2020-25695 - CVE-2021-23214 - - - + 300ff0bbf795643e914b7c8a6d6ba812a8354d62 + CVE-2015-0241 + CVE-2015-0242 + CVE-2015-0243 + CVE-2015-0244 + CVE-2015-3166 + CVE-2015-3167 + CVE-2016-0766 + CVE-2016-0768 + CVE-2016-0773 + CVE-2016-5423 + CVE-2016-5424 + CVE-2016-7048 + CVE-2017-14798 + CVE-2017-7484 + CVE-2018-1115 + CVE-2019-10127 + CVE-2019-10128 + CVE-2019-10210 + CVE-2019-10211 + CVE-2020-25694 + CVE-2020-25695 + CVE-2021-23214 + + + file name: protostream-types-4.4.1.Final.jar - ]]> - 29b45ebea1e4ce62ab3ec5eb76fa9771f98941b0 - CVE-2016-0750 - CVE-2017-15089 - CVE-2017-2638 - CVE-2019-10158 - CVE-2019-10174 - CVE-2020-25711 - - - + 29b45ebea1e4ce62ab3ec5eb76fa9771f98941b0 + CVE-2016-0750 + CVE-2017-15089 + CVE-2017-2638 + CVE-2019-10158 + CVE-2019-10174 + CVE-2020-25711 + + + file name: mariadb-java-client-2.7.5.jar - ]]> - 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 - CVE-2020-28912 - CVE-2021-46669 - CVE-2021-46666 - CVE-2021-46667 - - - - + 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 + CVE-2020-28912 + CVE-2021-46669 + CVE-2021-46666 + CVE-2021-46667 + + + + file name: cassandra-driver-core-3.11.2.jar - ]]> - e0aad9f8611e710b9a0ce49747f7465ce07d8404 - CVE-2020-17516 - CVE-2021-44521 - - - + e0aad9f8611e710b9a0ce49747f7465ce07d8404 + CVE-2020-17516 + CVE-2021-44521 + + + The vulnerable method is deprecated in Guava, but isn't removed. It's necessary to suppress this CVE. See https://github.com/google/guava/issues/4011 - ]]> - CVE-2020-8908 - - - - + CVE-2020-8908 + + + This is a false positive in avro-protobuf. The vulnerability is in Hamba avro golang library. - ]]> - CVE-2023-37475 - - - + CVE-2023-37475 + + + This CVE can be suppressed since it is covered in Pulsar by hostname verification changes made in https://github.com/apache/pulsar/pull/15824. - ]]> - CVE-2023-4586 - + + CVE-2023-4586 + diff --git a/src/settings.xml b/src/settings.xml index 80ec2f40620e7..8fc2f13a37a59 100644 --- a/src/settings.xml +++ b/src/settings.xml @@ -1,3 +1,4 @@ + apache.releases.https @@ -34,7 +34,6 @@ ${env.APACHE_PASSWORD} - apache diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index 8111ab34d72be..2fbed8f2dd793 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - structured-event-log Structured event logger - org.slf4j slf4j-api - org.apache.logging.log4j @@ -60,7 +56,6 @@ test - @@ -78,5 +73,4 @@ - diff --git a/testmocks/pom.xml b/testmocks/pom.xml index 73990cc8b462e..6333b5a265262 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - pulsar - org.apache.pulsar - 3.0.3-SNAPSHOT + io.streamnative + 3.0.0-SNAPSHOT - testmocks jar Pulsar Test Mocks - - org.apache.zookeeper zookeeper @@ -52,29 +48,23 @@ - org.apache.bookkeeper bookkeeper-server - org.apache.commons commons-lang3 - org.testng testng - org.objenesis objenesis - - @@ -92,5 +82,4 @@ - diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 01d70207e2f74..d8978a381e75c 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.0.3-SNAPSHOT - - - bc_2_0_0 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.0-rc1-incubating - - - - - org.apache.pulsar - pulsar-client - 2.0.0-rc1-incubating - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.0.0-SNAPSHOT + + bc_2_0_0 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.0-rc1-incubating + + + org.apache.pulsar + pulsar-client + 2.0.0-rc1-incubating + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/bc_2_0_0/src/test/resources/pulsar.xml b/tests/bc_2_0_0/src/test/resources/pulsar.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_0_0/src/test/resources/pulsar.xml +++ b/tests/bc_2_0_0/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 693f4e9efabdf..449faeb422b51 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.0.3-SNAPSHOT - - - bc_2_0_1 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.1-incubating - - - - - org.apache.pulsar - pulsar-client - 2.0.1-incubating - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.0.0-SNAPSHOT + + bc_2_0_1 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.1-incubating + + + org.apache.pulsar + pulsar-client + 2.0.1-incubating + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/bc_2_0_1/src/test/resources/pulsar.xml b/tests/bc_2_0_1/src/test/resources/pulsar.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_0_1/src/test/resources/pulsar.xml +++ b/tests/bc_2_0_1/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 075830fe7db57..7cd7b8ef6d889 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -1,4 +1,4 @@ - + - - - org.apache.pulsar.tests - tests-parent - 3.0.3-SNAPSHOT - - 4.0.0 - - bc_2_6_0 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.6.0 - - - - - com.google.code.gson - gson - test - - - - org.apache.pulsar - pulsar-client - 2.6.0 - test - - - - org.testcontainers - testcontainers - test - - - - - + + + io.streamnative.tests + tests-parent + 3.0.0-SNAPSHOT + + 4.0.0 + bc_2_6_0 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.6.0 + + + com.google.code.gson + gson + test + + + org.apache.pulsar + pulsar-client + 2.6.0 + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/backwards-client.xml - - 1 - - - - - - - - + false + + src/test/resources/backwards-client.xml + + 1 + + + + + + diff --git a/tests/bc_2_6_0/src/test/resources/backwards-client.xml b/tests/bc_2_6_0/src/test/resources/backwards-client.xml index 43dfaea15813c..2e9d57ed72832 100644 --- a/tests/bc_2_6_0/src/test/resources/backwards-client.xml +++ b/tests/bc_2_6_0/src/test/resources/backwards-client.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index f42f86fbba7db..8625bbfc48bd1 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 java-test-functions Apache Pulsar :: Tests :: Docker Images :: Java Test Functions - org.apache.pulsar + io.streamnative pulsar-io-core ${project.version} - org.apache.pulsar + io.streamnative pulsar-functions-api ${project.version} @@ -59,7 +59,6 @@ jar - docker @@ -68,15 +67,13 @@ integrationTests - - org.apache.pulsar + io.streamnative pulsar-functions-api-examples ${project.version} - @@ -91,7 +88,7 @@ - org.apache.pulsar:pulsar-functions-api-examples + io.streamnative:pulsar-functions-api-examples @@ -102,5 +99,4 @@ - diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index cb0336d3048ba..e063b11aad4a1 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 java-test-image Apache Pulsar :: Tests :: Docker Images :: Java Test Image pom - docker @@ -45,12 +44,12 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} - org.apache.pulsar + io.streamnative pulsar-server-distribution ${project.parent.version} bin @@ -78,7 +77,7 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} jar @@ -87,7 +86,7 @@ java-test-functions.jar - org.apache.pulsar + io.streamnative pulsar-server-distribution ${project.parent.version} bin diff --git a/tests/docker-images/java-test-plugins/pom.xml b/tests/docker-images/java-test-plugins/pom.xml index e6a5c5aca62ef..f55aabad0f47e 100644 --- a/tests/docker-images/java-test-plugins/pom.xml +++ b/tests/docker-images/java-test-plugins/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 java-test-plugins @@ -31,7 +31,7 @@ jar - org.apache.pulsar + io.streamnative pulsar-broker ${project.version} provided @@ -45,5 +45,4 @@ - diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 5d4bf9e7711d6..769a2c972950b 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT 4.0.0 latest-version-image Apache Pulsar :: Tests :: Docker Images :: Latest Version Testing pom - docker @@ -40,17 +39,17 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} - org.apache.pulsar.tests + io.streamnative.tests java-test-plugins ${project.parent.version} - org.apache.pulsar + io.streamnative pulsar-all-docker-image ${project.parent.version} pom @@ -77,7 +76,7 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} jar @@ -86,7 +85,7 @@ java-test-functions.jar - org.apache.pulsar.tests + io.streamnative.tests java-test-plugins ${project.parent.version} jar diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index f2a77faa9211c..fe29786515781 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar.tests + io.streamnative.tests tests-parent - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 540c32092cfd9..71d9d97b96f9c 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar.tests + io.streamnative.tests tests-parent - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - integration jar Apache Pulsar :: Tests :: Integration - pulsar.xml 4.1.2 - com.google.code.gson @@ -44,37 +40,37 @@ test - org.apache.pulsar + io.streamnative pulsar-functions-api-examples ${project.version} test - org.apache.pulsar + io.streamnative pulsar-broker ${project.version} test - org.apache.pulsar + io.streamnative pulsar-common ${project.version} test - org.apache.pulsar + io.streamnative pulsar-client-original ${project.version} test - org.apache.pulsar + io.streamnative pulsar-client-admin-original ${project.version} test - org.apache.pulsar + io.streamnative managed-ledger ${project.version} test @@ -96,31 +92,27 @@ test - org.apache.pulsar + io.streamnative pulsar-io-kafka ${project.version} test - org.testcontainers mysql test - org.postgresql postgresql ${postgresql-jdbc.version} runtime - org.testcontainers postgresql test - com.github.docker-java docker-java-core @@ -131,72 +123,60 @@ bcpkix-jdk18on test - - org.apache.pulsar + io.streamnative pulsar-io-jdbc-postgres ${project.version} test - com.fasterxml.jackson.core jackson-databind test - com.fasterxml.jackson.dataformat jackson-dataformat-yaml test - org.opensearch.client opensearch-rest-high-level-client test - co.elastic.clients elasticsearch-java test - org.testcontainers elasticsearch test - - com.rabbitmq amqp-client ${rabbitmq-client.version} test - joda-time joda-time test - io.trino trino-jdbc ${trino.version} test - org.awaitility awaitility test - - + org.testcontainers localstack @@ -213,7 +193,6 @@ aws-java-sdk-core test - org.testcontainers @@ -226,9 +205,7 @@ ${mongo-reactivestreams.version} test - - @@ -269,7 +246,6 @@ - integrationTests diff --git a/tests/integration/src/test/resources/pulsar-auth.xml b/tests/integration/src/test/resources/pulsar-auth.xml index 81d2c1361a269..3647730dee201 100644 --- a/tests/integration/src/test/resources/pulsar-auth.xml +++ b/tests/integration/src/test/resources/pulsar-auth.xml @@ -1,3 +1,4 @@ + - + - - + + - + diff --git a/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml b/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml index b50af74ed17f0..b0f55f3e04b16 100644 --- a/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml +++ b/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-cli.xml b/tests/integration/src/test/resources/pulsar-cli.xml index af55aca8a0098..1243dc7e485d7 100644 --- a/tests/integration/src/test/resources/pulsar-cli.xml +++ b/tests/integration/src/test/resources/pulsar-cli.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-function.xml b/tests/integration/src/test/resources/pulsar-function.xml index a18a65ff2e16d..03a8d83997a3c 100644 --- a/tests/integration/src/test/resources/pulsar-function.xml +++ b/tests/integration/src/test/resources/pulsar-function.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-ora-source.xml b/tests/integration/src/test/resources/pulsar-io-ora-source.xml index 1c5bb5faf677c..0a9927015be86 100644 --- a/tests/integration/src/test/resources/pulsar-io-ora-source.xml +++ b/tests/integration/src/test/resources/pulsar-io-ora-source.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-sinks.xml b/tests/integration/src/test/resources/pulsar-io-sinks.xml index 614c371f3e52c..046f977f3e13b 100644 --- a/tests/integration/src/test/resources/pulsar-io-sinks.xml +++ b/tests/integration/src/test/resources/pulsar-io-sinks.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-sources.xml b/tests/integration/src/test/resources/pulsar-io-sources.xml index 636b3e479195f..c995a7df249dd 100644 --- a/tests/integration/src/test/resources/pulsar-io-sources.xml +++ b/tests/integration/src/test/resources/pulsar-io-sources.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - \ No newline at end of file + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-loadbalance.xml b/tests/integration/src/test/resources/pulsar-loadbalance.xml index dfc4536e25592..d28dd1e0da7ee 100644 --- a/tests/integration/src/test/resources/pulsar-loadbalance.xml +++ b/tests/integration/src/test/resources/pulsar-loadbalance.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-messaging.xml b/tests/integration/src/test/resources/pulsar-messaging.xml index cfbdb22587034..f8813b614f5cb 100644 --- a/tests/integration/src/test/resources/pulsar-messaging.xml +++ b/tests/integration/src/test/resources/pulsar-messaging.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-plugin.xml b/tests/integration/src/test/resources/pulsar-plugin.xml index f88b67256e547..06fa8ee628c0f 100644 --- a/tests/integration/src/test/resources/pulsar-plugin.xml +++ b/tests/integration/src/test/resources/pulsar-plugin.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-process.xml b/tests/integration/src/test/resources/pulsar-process.xml index 8e5258d30624c..a6f1ac468d954 100644 --- a/tests/integration/src/test/resources/pulsar-process.xml +++ b/tests/integration/src/test/resources/pulsar-process.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-proxy.xml b/tests/integration/src/test/resources/pulsar-proxy.xml index ae6f13810535c..fabfb54d781a3 100644 --- a/tests/integration/src/test/resources/pulsar-proxy.xml +++ b/tests/integration/src/test/resources/pulsar-proxy.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-python.xml b/tests/integration/src/test/resources/pulsar-python.xml index a5faa6389e0f1..365768083d21b 100644 --- a/tests/integration/src/test/resources/pulsar-python.xml +++ b/tests/integration/src/test/resources/pulsar-python.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-schema.xml b/tests/integration/src/test/resources/pulsar-schema.xml index e07fdf2b2d86f..595bb8fda5521 100644 --- a/tests/integration/src/test/resources/pulsar-schema.xml +++ b/tests/integration/src/test/resources/pulsar-schema.xml @@ -1,3 +1,4 @@ + - + - - - - - - - + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-semantics.xml b/tests/integration/src/test/resources/pulsar-semantics.xml index 5b5402af4623b..69e171931f0e4 100644 --- a/tests/integration/src/test/resources/pulsar-semantics.xml +++ b/tests/integration/src/test/resources/pulsar-semantics.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-sql.xml b/tests/integration/src/test/resources/pulsar-sql.xml index 1ab4d479ad041..a7e0995f1f2d0 100644 --- a/tests/integration/src/test/resources/pulsar-sql.xml +++ b/tests/integration/src/test/resources/pulsar-sql.xml @@ -1,3 +1,4 @@ + - + - - - - - - - + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-standalone.xml b/tests/integration/src/test/resources/pulsar-standalone.xml index d8892c0746181..45d79a72821b5 100644 --- a/tests/integration/src/test/resources/pulsar-standalone.xml +++ b/tests/integration/src/test/resources/pulsar-standalone.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-thread.xml b/tests/integration/src/test/resources/pulsar-thread.xml index cf3da15e8e1ad..dc1b8e3b3de15 100644 --- a/tests/integration/src/test/resources/pulsar-thread.xml +++ b/tests/integration/src/test/resources/pulsar-thread.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - \ No newline at end of file + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-tls.xml b/tests/integration/src/test/resources/pulsar-tls.xml index 153d14b62a725..88e90c43107fb 100644 --- a/tests/integration/src/test/resources/pulsar-tls.xml +++ b/tests/integration/src/test/resources/pulsar-tls.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-transaction.xml b/tests/integration/src/test/resources/pulsar-transaction.xml index 72c375d000d8e..425f58ebda639 100644 --- a/tests/integration/src/test/resources/pulsar-transaction.xml +++ b/tests/integration/src/test/resources/pulsar-transaction.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-upgrade.xml b/tests/integration/src/test/resources/pulsar-upgrade.xml index a52db54753372..cb312b04ed7a3 100644 --- a/tests/integration/src/test/resources/pulsar-upgrade.xml +++ b/tests/integration/src/test/resources/pulsar-upgrade.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-websocket.xml b/tests/integration/src/test/resources/pulsar-websocket.xml index 87bf832d4e40a..e5023451518e0 100644 --- a/tests/integration/src/test/resources/pulsar-websocket.xml +++ b/tests/integration/src/test/resources/pulsar-websocket.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index d309a7695dd0b..ada31c2e5da8c 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/tiered-filesystem-storage.xml b/tests/integration/src/test/resources/tiered-filesystem-storage.xml index b14077594e0c9..c8cb694ea2d02 100644 --- a/tests/integration/src/test/resources/tiered-filesystem-storage.xml +++ b/tests/integration/src/test/resources/tiered-filesystem-storage.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/tiered-jcloud-storage.xml b/tests/integration/src/test/resources/tiered-jcloud-storage.xml index f61f9f5fa1612..be699ba0a8680 100644 --- a/tests/integration/src/test/resources/tiered-jcloud-storage.xml +++ b/tests/integration/src/test/resources/tiered-jcloud-storage.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/pom.xml b/tests/pom.xml index 2e26b1c6606a8..2db89a5a8465f 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT - org.apache.pulsar.tests + io.streamnative.tests tests-parent Apache Pulsar :: Tests @@ -50,7 +49,7 @@ skipIntegrationTests - + integrationTests diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 1cf518d0002ba..f6f92e3402e43 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.0.3-SNAPSHOT - - - pulsar-client-admin-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-Admin-Shade Test - - - - - org.apache.pulsar - pulsar-client-admin - ${project.version} - test - - - - org.apache.pulsar - pulsar-client-admin-api - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - org.apache.pulsar - pulsar-client-messagecrypto-bc - ${project.version} - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.0.0-SNAPSHOT + + pulsar-client-admin-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-Admin-Shade Test + + + io.streamnative + pulsar-client-admin + ${project.version} + test + + + io.streamnative + pulsar-client-admin-api + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml index d95dd95ae1775..2476755a95f7e 100644 --- a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 4e6d0902bfa62..6c0b5ae9b6b04 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 3.0.3-SNAPSHOT - - - pulsar-client-all-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test - - - - - org.apache.pulsar - pulsar-client-all - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - org.apache.pulsar - bouncy-castle-bc - ${project.version} - pkg - - - - org.apache.pulsar - pulsar-client-messagecrypto-bc - ${project.version} - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.0.0-SNAPSHOT + + pulsar-client-all-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test + + + io.streamnative + pulsar-client-all + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + io.streamnative + bouncy-castle-bc + ${project.version} + pkg + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml index 5c725f80eaea6..cfbc5ce59488b 100644 --- a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index 693653541fce0..4ae9800292445 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar.tests - tests-parent - 3.0.3-SNAPSHOT - - - pulsar-client-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-Shade Test - - - - - org.apache.pulsar - pulsar-client - ${project.version} - test - - - - org.apache.pulsar - pulsar-client-admin - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 3.0.0-SNAPSHOT + + pulsar-client-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-Shade Test + + + io.streamnative + pulsar-client + ${project.version} + test + + + io.streamnative + pulsar-client-admin + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml index 6132ad05f6ccf..e2ac6a9ee4f30 100644 --- a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 92e955aac4894..9ce9353d9d4fc 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - tiered-storage-parent - 3.0.3-SNAPSHOT - .. - - - tiered-storage-file-system - Apache Pulsar :: Tiered Storage :: File System - - - ${project.groupId} - managed-ledger - ${project.version} - - - org.apache.hadoop - hadoop-common - ${hdfs-offload-version3} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hadoop - hadoop-hdfs-client - ${hdfs-offload-version3} - - - org.apache.avro - avro - - - org.mortbay.jetty - jetty - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - javax.servlet - servlet-api - - - - - - org.apache.avro - avro - ${avro.version} - - - - net.minidev - json-smart - ${json-smart.version} - - - com.google.protobuf - protobuf-java - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.apache.hadoop - hadoop-minicluster - ${hdfs-offload-version3} - test - - - io.netty - netty-all - - - org.bouncycastle - * - - - - - - org.bouncycastle - bcpkix-jdk18on - test - - - + + 4.0.0 + + io.streamnative + tiered-storage-parent + 3.0.0-SNAPSHOT + .. + + tiered-storage-file-system + Apache Pulsar :: Tiered Storage :: File System + + + ${project.groupId} + managed-ledger + ${project.version} + + + org.apache.hadoop + hadoop-common + ${hdfs-offload-version3} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hdfs-offload-version3} + + + org.apache.avro + avro + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + javax.servlet + servlet-api + + + + + + org.apache.avro + avro + ${avro.version} + + + + net.minidev + json-smart + ${json-smart.version} + + + com.google.protobuf + protobuf-java + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.apache.hadoop + hadoop-minicluster + ${hdfs-offload-version3} + test + + io.netty - netty-codec-http - - - - org.eclipse.jetty - jetty-server - test - - - org.eclipse.jetty - jetty-alpn-conscrypt-server - test - - - org.eclipse.jetty - jetty-servlet - test - - - org.eclipse.jetty - jetty-util - test - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/tiered-storage/file-system/src/main/resources/findbugsExclude.xml b/tiered-storage/file-system/src/main/resources/findbugsExclude.xml index 051f0e68e257a..1b8cc1ac799ca 100644 --- a/tiered-storage/file-system/src/main/resources/findbugsExclude.xml +++ b/tiered-storage/file-system/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative tiered-storage-parent - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - tiered-storage-jcloud Apache Pulsar :: Tiered Storage :: JCloud - ${project.groupId} managed-ledger ${project.version} - ${project.groupId} jclouds-shaded @@ -74,7 +70,6 @@ - org.apache.jclouds jclouds-allblobstore @@ -85,12 +80,10 @@ com.amazonaws aws-java-sdk-core - com.amazonaws aws-java-sdk-sts - ${project.groupId} testmocks @@ -103,7 +96,6 @@ ${jclouds.version} provided - javax.xml.bind jaxb-api @@ -115,13 +107,11 @@ runtime - com.sun.activation javax.activation runtime - @@ -129,7 +119,6 @@ org.apache.nifi nifi-nar-maven-plugin - com.github.spotbugs spotbugs-maven-plugin diff --git a/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml b/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml index a8a9b9aae925f..c6ad70823a5ac 100644 --- a/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml +++ b/tiered-storage/jcloud/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 3.0.3-SNAPSHOT + 3.0.0-SNAPSHOT .. - tiered-storage-parent Apache Pulsar :: Tiered Storage :: Parent - ${project.version} - jcloud file-system