From 641df068875f1c118e3dd0dc2f204c2ed2ef8cbe Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 02:21:04 -0800 Subject: [PATCH 01/14] Move pulsar functions dependency version to root pom and remove duplicated license headers This addresses some comments in pulsar functions PR #1314 --- pom.xml | 37 ++++ pulsar-functions/api-java/pom.xml | 1 - pulsar-functions/cli/pom.xml | 6 - pulsar-functions/instance/pom.xml | 139 +-------------- .../instance/src/main/python/contextimpl.py | 17 -- .../instance/src/main/python/log.py | 17 -- .../src/main/python/python_instance.py | 17 -- .../src/main/python/python_instance_main.py | 17 -- .../instance/src/main/python/server.py | 17 -- .../instance/src/main/python/util.py | 17 -- pulsar-functions/metrics/pom.xml | 56 ------ pulsar-functions/proto/pom.xml | 7 +- pulsar-functions/runtime/pom.xml | 163 ------------------ pulsar-functions/utils/pom.xml | 21 --- pulsar-functions/worker/pom.xml | 32 ---- 15 files changed, 46 insertions(+), 518 deletions(-) diff --git a/pom.xml b/pom.xml index c2c10d1d51ad7..e44b2b530e964 100644 --- a/pom.xml +++ b/pom.xml @@ -129,6 +129,12 @@ flexible messaging model and an intuitive client API. 2.8.4 6.19 1.3.7 + 0.5.0 + 3.5.1 + 1.5.0 + 1.0.0 + 2.8.2 + 0.6.0 3.4.0 @@ -619,6 +625,37 @@ flexible messaging model and an intuitive client API. ${bouncycastle.version} + + net.jodah + typetools + ${typetools.version} + + + + io.grpc + grpc-all + ${grpc.version} + + + + com.google.code.gson + gson + ${gson.version} + + + + com.yahoo.datasketches + sketches-core + ${sketches.version} + + + + + org.apache.distributedlog + distributedlog-core-shaded + ${bookkeeper.version} + + com.lmax diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 9d47ee3c9d04c..480d19a9f463a 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -40,7 +40,6 @@ net.jodah typetools - 0.5.0 test diff --git a/pulsar-functions/cli/pom.xml b/pulsar-functions/cli/pom.xml index 5e8f76b6b7c26..b22a5022de259 100644 --- a/pulsar-functions/cli/pom.xml +++ b/pulsar-functions/cli/pom.xml @@ -53,12 +53,6 @@ ${project.version} - - ${project.groupId} - pulsar-client-tools-shaded - ${project.version} - - org.glassfish.jersey.media jersey-media-json-jackson diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index e0bcb53855d04..fb45cdfdf3d87 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -51,154 +51,22 @@ log4j-core - - ${project.groupId} - pulsar-client-tools-shaded - ${project.version} - - - - org.apache.commons - commons-lang3 - - - commons-codec - commons-codec - - - commons-collections - commons-collections - - - org.asynchttpclient - async-http-client - - - io.netty - netty - - - io.netty - netty-all - - - io.netty - netty-codec-http - - - io.netty - netty-tcnative-boringssl-static - - - io.netty - netty-transport-native-epoll - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - com.google.code.gson - gson - - - net.jpountz.lz4 - lz4 - - - org.apache.httpcomponents - httpcomponents-core - - - org.apache.httpcomponents - httpcomponents-client - - - commons-logging - commons-logging - - - com.yahoo.datasketches - sketches-core - - - - - - org.apache.pulsar - pulsar-functions-proto - ${project.version} - - - - ${project.groupId} - pulsar-functions-api - ${project.version} - - ${project.groupId} pulsar-functions-utils ${project.version} - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson.version} - - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - - com.google.code.gson - gson - ${gson.version} - - - - - com.google.protobuf - protobuf-java - 3.5.1 - - - - io.grpc - grpc-all - 1.5.0 - - + org.apache.bookkeeper stream-storage-java-client 0.6.0 - - net.jodah - typetools - 0.5.0 - - com.yahoo.datasketches sketches-core - 0.6.0 @@ -211,6 +79,11 @@ jcommander + + net.jodah + typetools + + diff --git a/pulsar-functions/instance/src/main/python/contextimpl.py b/pulsar-functions/instance/src/main/python/contextimpl.py index 9218407b96c46..b021421505444 100644 --- a/pulsar-functions/instance/src/main/python/contextimpl.py +++ b/pulsar-functions/instance/src/main/python/contextimpl.py @@ -20,23 +20,6 @@ # -*- encoding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# """contextimpl.py: ContextImpl class that implements the Context interface """ diff --git a/pulsar-functions/instance/src/main/python/log.py b/pulsar-functions/instance/src/main/python/log.py index 3838b60e05e97..f36e684f23efd 100644 --- a/pulsar-functions/instance/src/main/python/log.py +++ b/pulsar-functions/instance/src/main/python/log.py @@ -20,23 +20,6 @@ # -*- encoding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ''' log.py ''' import logging from logging.handlers import RotatingFileHandler diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index ed5f57342abf0..08fcef5272869 100644 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -20,23 +20,6 @@ # -*- encoding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# """python_instance.py: Python Instance for running python functions """ import os diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py index 9d4159d8dc81f..3fd43580ae18c 100644 --- a/pulsar-functions/instance/src/main/python/python_instance_main.py +++ b/pulsar-functions/instance/src/main/python/python_instance_main.py @@ -20,23 +20,6 @@ # -*- encoding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# """python_instance_main.py: The main for the Python Instance """ import argparse diff --git a/pulsar-functions/instance/src/main/python/server.py b/pulsar-functions/instance/src/main/python/server.py index 816d5b732f91a..3fe7bd44ac4ed 100644 --- a/pulsar-functions/instance/src/main/python/server.py +++ b/pulsar-functions/instance/src/main/python/server.py @@ -20,23 +20,6 @@ # -*- encoding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# """python_instance.py: Python Instance for running python functions """ from concurrent import futures diff --git a/pulsar-functions/instance/src/main/python/util.py b/pulsar-functions/instance/src/main/python/util.py index 439f648e4da38..efa238dee4762 100644 --- a/pulsar-functions/instance/src/main/python/util.py +++ b/pulsar-functions/instance/src/main/python/util.py @@ -20,23 +20,6 @@ # -*- encoding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# """util.py: Some misc utility functions """ import os diff --git a/pulsar-functions/metrics/pom.xml b/pulsar-functions/metrics/pom.xml index 6c890a3e145c2..421b4ecdc2f97 100644 --- a/pulsar-functions/metrics/pom.xml +++ b/pulsar-functions/metrics/pom.xml @@ -22,10 +22,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - 2.8.2 - - org.apache.pulsar pulsar-functions @@ -37,20 +33,6 @@ - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - org.apache.pulsar pulsar-functions-proto @@ -63,44 +45,6 @@ ${project.version} - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson.version} - - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - - com.google.code.gson - gson - ${gson.version} - - - - - com.google.protobuf - protobuf-java - 3.5.1 - - - - net.jodah - typetools - 0.5.0 - - diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index 03b19022bbd5d..9bae8b52ba0e8 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -38,13 +38,12 @@ com.google.protobuf protobuf-java - 3.5.1 + ${protobuf3.version} io.grpc grpc-all - 1.5.0 @@ -62,10 +61,10 @@ protobuf-maven-plugin 0.5.0 - com.google.protobuf:protoc:3.5.1:exe:${os.detected.classifier} + com.google.protobuf:protoc:${protobuf3.version}:exe:${os.detected.classifier} true grpc-java - io.grpc:protoc-gen-grpc-java:1.0.0:exe:${os.detected.classifier} + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index 4a491621bf61e..dc236b6736341 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -22,10 +22,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - 2.8.2 - - org.apache.pulsar pulsar-functions @@ -37,109 +33,6 @@ - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - - ${project.groupId} - pulsar-client-tools-shaded - ${project.version} - - - - org.apache.commons - commons-lang3 - - - commons-codec - commons-codec - - - commons-collections - commons-collections - - - org.asynchttpclient - async-http-client - - - io.netty - netty - - - io.netty - netty-all - - - io.netty - netty-codec-http - - - io.netty - netty-tcnative-boringssl-static - - - io.netty - netty-transport-native-epoll - - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - - - com.google.code.gson - gson - - - net.jpountz.lz4 - lz4 - - - org.apache.httpcomponents - httpcomponents-core - - - org.apache.httpcomponents - httpcomponents-client - - - commons-logging - commons-logging - - - com.yahoo.datasketches - sketches-core - - - - - - org.apache.pulsar - pulsar-functions-proto - ${project.version} - - - - ${project.groupId} - pulsar-functions-api - ${project.version} - - org.apache.pulsar pulsar-functions-instance @@ -152,62 +45,6 @@ ${project.version} - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.fasterxml.jackson.core - jackson-annotations - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.google.code.gson - gson - ${gson.version} - - - - - com.google.protobuf - protobuf-java - 3.5.1 - - - - io.grpc - grpc-all - 1.5.0 - - - - net.jodah - typetools - 0.5.0 - - - - com.yahoo.datasketches - sketches-core - 0.6.0 - - - - com.google.guava - guava - - - - com.beust - jcommander - - diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index 0568d5cccf9a3..b71931d90b9a8 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -22,10 +22,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - 2.8.2 - - org.apache.pulsar pulsar-functions @@ -143,38 +139,21 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - ${jackson.version} com.fasterxml.jackson.core jackson-annotations - ${jackson.version} com.fasterxml.jackson.core jackson-databind - ${jackson.version} com.google.code.gson gson - ${gson.version} - - - - - com.google.protobuf - protobuf-java - 3.5.1 - - - - net.jodah - typetools - 0.5.0 diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index b653304d4dda5..24f690f4c41d0 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -39,41 +39,21 @@ org.apache.logging.log4j log4j-slf4j-impl - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - - org.apache.pulsar - pulsar-client-tools-shaded - ${project.version} - - ${project.groupId} pulsar-functions-runtime ${project.version} - - org.apache.pulsar - pulsar-functions-proto - ${project.version} - - - - ${project.groupId} - pulsar-functions-metrics - ${project.version} - - org.glassfish.jersey.media jersey-media-json-jackson @@ -109,22 +89,10 @@ jetty-servlet - - com.beust - jcommander - - - - org.inferred - freebuilder - 1.14.6 - - org.apache.distributedlog distributedlog-core-shaded - ${bookkeeper.version} From ef65a9f33456e0340bd4464dd262d6103d266a37 Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 11:40:19 -0800 Subject: [PATCH 02/14] shade worker --- buildtools/pom.xml | 15 +++ managed-ledger/pom.xml | 15 --- pom.xml | 36 +++---- pulsar-functions/bin/pulsar | 6 +- pulsar-functions/pom.xml | 2 + pulsar-functions/worker-runner/pom.xml | 106 +++++++++++++++++++++ pulsar-functions/worker-shaded/pom.xml | 126 +++++++++++++++++++++++++ pulsar-zookeeper-utils/pom.xml | 15 --- 8 files changed, 272 insertions(+), 49 deletions(-) create mode 100644 pulsar-functions/worker-runner/pom.xml create mode 100644 pulsar-functions/worker-shaded/pom.xml diff --git a/buildtools/pom.xml b/buildtools/pom.xml index ae217e4da4205..19bf1df86956a 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -34,6 +34,21 @@ testng 6.13.1 + + org.apache.logging.log4j + log4j-api + 2.10.0 + + + org.apache.logging.log4j + log4j-core + 2.10.0 + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.10.0 + diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 3cd0726a59892..b4f0017fa9b79 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -36,21 +36,6 @@ org.apache.bookkeeper bookkeeper-server-shaded - ${bookkeeper.version} - - - slf4j-log4j12 - org.slf4j - - - log4j - log4j - - - org.jboss.netty - netty - - diff --git a/pom.xml b/pom.xml index c2c10d1d51ad7..da09947aedb4b 100644 --- a/pom.xml +++ b/pom.xml @@ -224,6 +224,26 @@ flexible messaging model and an intuitive client API. + + org.apache.bookkeeper + bookkeeper-server-shaded + ${bookkeeper.version} + + + slf4j-log4j12 + org.slf4j + + + log4j + log4j + + + org.jboss.netty + netty + + + + org.apache.bookkeeper bookkeeper-bookkeeper-stats-api @@ -638,22 +658,6 @@ flexible messaging model and an intuitive client API. test - - org.apache.logging.log4j - log4j-api - test - - - org.apache.logging.log4j - log4j-core - test - - - org.apache.logging.log4j - log4j-slf4j-impl - test - - org.testng testng diff --git a/pulsar-functions/bin/pulsar b/pulsar-functions/bin/pulsar index 4c35fdc9d7c86..ba219a1b617e3 100755 --- a/pulsar-functions/bin/pulsar +++ b/pulsar-functions/bin/pulsar @@ -52,7 +52,7 @@ fi # exclude tests jar if [ -z "$PULSAR_JAR" ]; then - BUILT_JAR=`ls $PULSAR_HOME/worker/target/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` + BUILT_JAR=`ls $PULSAR_HOME/worker-runner/target/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` if [ -z "${BUILT_JAR}" ]; then echo "\nCouldn't find pulsar jar."; echo "Make sure you've run 'mvn package'\n"; @@ -118,10 +118,10 @@ add_maven_deps_to_classpath() { # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. - f="${PULSAR_HOME}/worker/target/classpath_shaded.txt" + f="${PULSAR_HOME}/worker-runner/target/classpath_shaded.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${PULSAR_HOME}/worker/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null + ${MVN} -f "${PULSAR_HOME}/worker-runner/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null fi PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"` } diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 47450a83198a2..2a6daac208740 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -39,6 +39,8 @@ instance runtime worker + worker-shaded + worker-runner cli java-examples dist diff --git a/pulsar-functions/worker-runner/pom.xml b/pulsar-functions/worker-runner/pom.xml new file mode 100644 index 0000000000000..f5c53a88e1a53 --- /dev/null +++ b/pulsar-functions/worker-runner/pom.xml @@ -0,0 +1,106 @@ + + + + 4.0.0 + + + org.apache.pulsar + pulsar-functions + 2.0.0-incubating-SNAPSHOT + .. + + + pulsar-functions-worker-runner + Pulsar Functions :: Worker Runner + + + + + ${project.groupId} + pulsar-functions-worker-shaded + ${project.parent.version} + + + com.google.protobuf + protobuf-lite + + + com.google.protobuf + protobuf-java + + + com.google.protobuf.nano + protobuf-javanano + + + com.google.protobuf + protobuf-java-util + + + org.apache.pulsar + pulsar-functions-proto + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-protobuf-lite + + + io.grpc + grpc-protobuf-nano + + + + org.apache.pulsar + pulsar-client-tools-shaded + + + + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + + + + + stream-storage-mvn-repo + https://raw.githubusercontent.com/sijie/artifacts/mvn-repo/ + + true + + + + diff --git a/pulsar-functions/worker-shaded/pom.xml b/pulsar-functions/worker-shaded/pom.xml new file mode 100644 index 0000000000000..c1eb52ac56d83 --- /dev/null +++ b/pulsar-functions/worker-shaded/pom.xml @@ -0,0 +1,126 @@ + + + + 4.0.0 + + + org.apache.pulsar + pulsar-functions + 2.0.0-incubating-SNAPSHOT + .. + + + pulsar-functions-worker-shaded + Pulsar Functions :: Worker Shaded + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + + ${project.groupId} + pulsar-functions-worker + ${project.parent.version} + + + + org.apache.bookkeeper + bookkeeper-common + + + + + + + org.apache.bookkeeper + bookkeeper-server-shaded + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + true + false + + + + com.google.protobuf:protobuf-lite + com.google.protobuf:protobuf-java + com.google.protobuf.nano:protobuf-javanano + + org.apache.pulsar:pulsar-functions-* + + com.google.protobuf:protobuf-java-util + io.grpc:grpc-protobuf + io.grpc:grpc-protobuf-lite + io.grpc:grpc-protobuf-nano + + + + + + com.google.protobuf + org.apache.pulsar.functions.shaded.com.google.protobuf + + + + + + + + + + + + + stream-storage-mvn-repo + https://raw.githubusercontent.com/sijie/artifacts/mvn-repo/ + + true + + + + diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index 0a621f03ef7dd..b8019a59eaddd 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -42,21 +42,6 @@ org.apache.bookkeeper bookkeeper-server-shaded - ${bookkeeper.version} - - - slf4j-log4j12 - org.slf4j - - - log4j - log4j - - - org.jboss.netty - netty - - From 7bc05f162d6d35effe041f3ddfde9848fee2ebca Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 11:52:57 -0800 Subject: [PATCH 03/14] Fix broken master --- pulsar-client/pom.xml | 1 - .../java/org/apache/pulsar/admin/cli/CmdFunctions.java | 2 +- .../java/org/apache/pulsar/admin/cli/FunctionsTool.java | 3 ++- .../apache/pulsar/client/admin/PulsarFunctionsAdmin.java | 7 ++++--- .../java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java | 3 ++- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 3ab403fbe8915..267c5e7d74bd7 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -62,7 +62,6 @@ com.yahoo.datasketches sketches-core - 0.6.0 diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index f31135e2f7401..97fbf7bc3a552 100644 --- a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -451,7 +451,7 @@ void runCmd() throws Exception { throw new RuntimeException("Missing arguments"); } - String serviceUrl = ((PulsarFunctionsAdmin) admin).getClientConf().getConfigurationData().getServiceUrl(); + String serviceUrl = ((PulsarFunctionsAdmin) admin).getClientConf().getServiceUrl(); if (brokerServiceUrl != null) { serviceUrl = brokerServiceUrl; } diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/FunctionsTool.java b/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/FunctionsTool.java index 9924330689bc1..f3eeaee46643a 100644 --- a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/FunctionsTool.java +++ b/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/FunctionsTool.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.Properties; import org.apache.pulsar.client.api.ClientConfiguration; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; /** * TODO: merge this into {@link PulsarAdminTool}. @@ -73,7 +74,7 @@ public static void main(String[] args) throws Exception { isLocalRun = "localrun" == args[cmdPos].toLowerCase(); } - BiFunction adminFactory; + BiFunction adminFactory; if (isLocalRun) { // bypass constructing admin client adminFactory = (url, config) -> null; diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/PulsarFunctionsAdmin.java b/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/PulsarFunctionsAdmin.java index f35473c83b5ea..43ede46808e30 100644 --- a/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/PulsarFunctionsAdmin.java +++ b/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/PulsarFunctionsAdmin.java @@ -21,6 +21,7 @@ import org.apache.pulsar.client.admin.internal.FunctionsImpl; import org.apache.pulsar.client.api.*; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +35,7 @@ public class PulsarFunctionsAdmin extends PulsarAdmin { private static final Logger LOG = LoggerFactory.getLogger(PulsarFunctionsAdmin.class); private final Functions functions; - private final ClientConfiguration clientConf; + private final ClientConfigurationData clientConf; /** * Construct a new Pulsar Admin client object. @@ -46,13 +47,13 @@ public class PulsarFunctionsAdmin extends PulsarAdmin { * @param pulsarConfig * the ClientConfiguration object to be used to talk with Pulsar */ - public PulsarFunctionsAdmin(URL serviceUrl, ClientConfiguration pulsarConfig) throws PulsarClientException { + public PulsarFunctionsAdmin(URL serviceUrl, ClientConfigurationData pulsarConfig) throws PulsarClientException { super(serviceUrl, pulsarConfig); this.functions = new FunctionsImpl(web, auth); this.clientConf = pulsarConfig; } - public ClientConfiguration getClientConf() { + public ClientConfigurationData getClientConf() { return clientConf; } diff --git a/pulsar-functions/cli/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-functions/cli/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index ef807f7a956ed..5f390f82ced34 100644 --- a/pulsar-functions/cli/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-functions/cli/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -54,6 +54,7 @@ import org.apache.pulsar.client.admin.Functions; import org.apache.pulsar.client.admin.PulsarFunctionsAdmin; import org.apache.pulsar.client.api.ClientConfiguration; +import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.api.PulsarFunction; import org.apache.pulsar.functions.api.utils.DefaultSerDe; @@ -104,7 +105,7 @@ public void setup() throws Exception { this.functions = mock(Functions.class); when(admin.functions()).thenReturn(functions); when(admin.getServiceUrl()).thenReturn(URI.create("http://localhost:1234").toURL()); - when(admin.getClientConf()).thenReturn(new ClientConfiguration()); + when(admin.getClientConf()).thenReturn(new ClientConfigurationData()); this.cmd = new CmdFunctions(admin); // mock reflections From fc8d4c53fddf8b1c03f1e0ab4fefed93d85ad6df Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 12:17:05 -0800 Subject: [PATCH 04/14] Upgrade the bookkeeper storage client dependency to the official bookkeeper version This removes the temp dependency in `pulsar-functions-instance` --- pom.xml | 6 ++++++ pulsar-functions/instance/pom.xml | 12 ------------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index c2c10d1d51ad7..f51c74a5a3162 100644 --- a/pom.xml +++ b/pom.xml @@ -224,6 +224,12 @@ flexible messaging model and an intuitive client API. + + org.apache.bookkeeper + stream-storage-java-client + ${bookkeeper.version} + + org.apache.bookkeeper bookkeeper-bookkeeper-stats-api diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index e0bcb53855d04..775a8f9f6cee9 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -186,7 +186,6 @@ org.apache.bookkeeper stream-storage-java-client - 0.6.0 @@ -213,15 +212,4 @@ - - - - stream-storage-mvn-repo - https://raw.githubusercontent.com/sijie/artifacts/mvn-repo/ - - true - - - - From 6b494c6179f9c6dd810fcfa08f1501ef75057b42 Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 13:26:25 -0800 Subject: [PATCH 05/14] set `protobuf2.version` in pulsar-common --- pom.xml | 1 + pulsar-common/pom.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 31bae59b014f8..95461c51a0afa 100644 --- a/pom.xml +++ b/pom.xml @@ -129,6 +129,7 @@ flexible messaging model and an intuitive client API. 2.8.4 6.19 1.3.7 + 2.4.1 3.4.0 diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index e227d6d899c2f..ee246f3d1e2cc 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -48,7 +48,7 @@ com.google.protobuf protobuf-java - 2.4.1 + ${protobuf2.version} From 59b10aa22d947756245a86596bde6f8c41272e43 Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 15:39:30 -0800 Subject: [PATCH 06/14] provide a shaded worker --- pulsar-functions/cli/pom.xml | 14 ------------ pulsar-functions/utils/pom.xml | 14 ------------ pulsar-functions/worker-runner/pom.xml | 17 +------------- pulsar-functions/worker-shaded/pom.xml | 31 +++++--------------------- pulsar-functions/worker/pom.xml | 14 ------------ 5 files changed, 6 insertions(+), 84 deletions(-) diff --git a/pulsar-functions/cli/pom.xml b/pulsar-functions/cli/pom.xml index b22a5022de259..2345e250782fc 100644 --- a/pulsar-functions/cli/pom.xml +++ b/pulsar-functions/cli/pom.xml @@ -33,20 +33,6 @@ - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - ${project.groupId} pulsar-functions-runtime diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index b71931d90b9a8..b5cb83e4c8bb4 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -33,20 +33,6 @@ - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - ${project.groupId} pulsar-client-tools-shaded diff --git a/pulsar-functions/worker-runner/pom.xml b/pulsar-functions/worker-runner/pom.xml index f5c53a88e1a53..3f79a2ebb235f 100644 --- a/pulsar-functions/worker-runner/pom.xml +++ b/pulsar-functions/worker-runner/pom.xml @@ -72,11 +72,6 @@ io.grpc grpc-protobuf-nano - - - org.apache.pulsar - pulsar-client-tools-shaded - @@ -91,16 +86,6 @@ pulsar-client-admin-original ${project.version} - - - - - stream-storage-mvn-repo - https://raw.githubusercontent.com/sijie/artifacts/mvn-repo/ - - true - - - + diff --git a/pulsar-functions/worker-shaded/pom.xml b/pulsar-functions/worker-shaded/pom.xml index c1eb52ac56d83..2e079db32ed6e 100644 --- a/pulsar-functions/worker-shaded/pom.xml +++ b/pulsar-functions/worker-shaded/pom.xml @@ -34,21 +34,6 @@ Pulsar Functions :: Worker Shaded - - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - ${project.groupId} pulsar-functions-worker @@ -59,6 +44,11 @@ org.apache.bookkeeper bookkeeper-common + + + org.apache.pulsar + pulsar-client-tools-shaded + @@ -112,15 +102,4 @@ - - - - - stream-storage-mvn-repo - https://raw.githubusercontent.com/sijie/artifacts/mvn-repo/ - - true - - - diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 24f690f4c41d0..6f408e7160978 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -34,20 +34,6 @@ - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - ${project.groupId} pulsar-functions-runtime From 0935c464af2f323a7ed4f6ab6e611135b116f6aa Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 15:50:45 -0800 Subject: [PATCH 07/14] include worker dependency at broker --- pulsar-broker/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 14dcca7ca274b..f4744aa943502 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -119,6 +119,12 @@ test + + ${project.groupId} + pulsar-functions-worker-shaded + ${project.version} + + org.eclipse.jetty jetty-server From e7daf2d2bb5eb72209162df343f3b688a99777e4 Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 21:34:17 -0800 Subject: [PATCH 08/14] Embeded function worker at broker --- bin/pulsar | 59 +- conf/function_worker.yml | 46 ++ conf/log4j2.yaml | 118 +++- pom.xml | 1 + pulsar-broker/pom.xml | 84 +++ .../apache/pulsar/PulsarBrokerStarter.java | 55 +- .../pulsar/PulsarStandaloneStarter.java | 42 +- .../apache/pulsar/broker/PulsarService.java | 12 + .../broker/admin/impl/FunctionsBase.java | 142 +++++ .../pulsar/broker/admin/v1/Functions.java | 28 + .../pulsar/broker/admin/v2/Functions.java | 28 + .../apache/pulsar/broker/web/WebService.java | 2 + pulsar-client-tools-test/pom.xml | 84 +++ .../pulsar/admin/cli/PulsarAdminToolTest.java | 0 .../pulsar/admin/cli/utils/IOUtilsTest.java | 0 .../utils/NameValueParameterSplitterTest.java | 0 .../client/cli/PulsarClientToolTest.java | 0 pulsar-client-tools/pom.xml | 133 ++--- pulsar-functions/instance/pom.xml | 1 - pulsar-functions/runtime/pom.xml | 5 + pulsar-functions/worker-runner/pom.xml | 28 +- pulsar-functions/worker-shaded/pom.xml | 18 +- .../functions/worker/MembershipManager.java | 2 +- .../pulsar/functions/worker/Worker.java | 11 +- .../functions/worker/WorkerService.java | 12 +- .../worker/rest/FunctionApiResource.java | 14 +- .../worker/rest/api/FunctionsImpl.java | 555 ++++++++++++++++++ .../rest/api/v2/FunctionApiV2Resource.java | 440 +------------- .../api/v2/FunctionApiV2ResourceTest.java | 7 +- 29 files changed, 1377 insertions(+), 550 deletions(-) create mode 100644 conf/function_worker.yml create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Functions.java create mode 100644 pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java create mode 100644 pulsar-client-tools-test/pom.xml rename {pulsar-client-tools => pulsar-client-tools-test}/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java (100%) rename {pulsar-client-tools => pulsar-client-tools-test}/src/test/java/org/apache/pulsar/admin/cli/utils/IOUtilsTest.java (100%) rename {pulsar-client-tools => pulsar-client-tools-test}/src/test/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitterTest.java (100%) rename {pulsar-client-tools => pulsar-client-tools-test}/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java (100%) create mode 100644 pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java diff --git a/bin/pulsar b/bin/pulsar index ed7e988c8cd77..0f12c302990f6 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -31,6 +31,14 @@ DEFAULT_STANDALONE_CONF=$PULSAR_HOME/conf/standalone.conf DEFAULT_WEBSOCKET_CONF=$PULSAR_HOME/conf/websocket.conf DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml +# functions related variables +FUNCTIONS_HOME=$PULSAR_HOME/pulsar-functions +DEFAULT_WORKER_CONF=$PULSAR_HOME/conf/function_worker.yml +DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar +JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"} +DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py +PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"} + if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] then . "$PULSAR_HOME/conf/pulsar_env.sh" @@ -63,6 +71,34 @@ elif [ -e "$BUILT_JAR" ]; then PULSAR_JAR=$BUILT_JAR fi +# +# find the instance locations for pulsar-functions +# + +# find the java instance location +if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then + # didn't find a released jar, then search the built jar + BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime/target/java-instance.jar" + if [ -z "${BUILT_JAVA_INSTANCE_JAR}" ]; then + echo "\nCouldn't find pulsar-functions java instance jar."; + echo "Make sure you've run 'mvn package'\n"; + exit 1; + fi + JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR} +fi + +# find the python instance location +if [ ! -f "${PY_INSTANCE_FILE}" ]; then + # didn't find a released python instance, then search the built python instance + BUILT_PY_INSTANCE_FILE="${FUNCTIONS_HOME}/runtime/target/python-instance/python_instance_main.py" + if [ -z "${BUILT_PY_INSTANCE_FILE}" ]; then + echo "\nCouldn't find pulsar-functions python instance."; + echo "Make sure you've run 'mvn package'\n"; + exit 1; + fi + PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE} +fi + pulsar_help() { cat < @@ -74,6 +110,7 @@ where command is one of: discovery Run a discovery server proxy Run a pulsar proxy websocket Run a web socket proxy server + worker Run a function worker server standalone Run a broker server with local bookies and local zookeeper compact-topic Run compaction against a topic @@ -93,6 +130,7 @@ Environment variables: PULSAR_DISCOVERY_CONF Configuration file for discovery service (default: $DEFAULT_DISCOVERY_CONF) PULSAR_WEBSOCKET_CONF Configuration file for websocket proxy (default: $DEFAULT_WEBSOCKET_CONF) PULSAR_PROXY_CONF Configuration file for Pulsar proxy (default: $DEFAULT_PROXY_CONF) + PULSAR_WORKER_CONF Configuration file for function worker (default: $DEFAULT_WORKER_CONF) PULSAR_STANDALONE_CONF Configuration file for standalone (default: $DEFAULT_STANDALONE_CONF) PULSAR_EXTRA_OPTS Extra options to be passed to the jvm PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath @@ -115,7 +153,7 @@ add_maven_deps_to_classpath() { f="${PULSAR_HOME}/all/target/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null fi PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"` } @@ -136,6 +174,10 @@ fi COMMAND=$1 shift +if [ -z "$PULSAR_WORKER_CONF" ]; then + PULSAR_WORKER_CONF=$DEFAULT_WORKER_CONF +fi + if [ -z "$PULSAR_BROKER_CONF" ]; then PULSAR_BROKER_CONF=$DEFAULT_BROKER_CONF fi @@ -186,12 +228,22 @@ OPTS="-cp $PULSAR_CLASSPATH $OPTS" OPTS="$OPTS $PULSAR_EXTRA_OPTS" # log directory & file -PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"} PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} +PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} +PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} +PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} #Configure log configuration system properties OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" +OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" +OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT" + +# Functions related logging +OPTS="$OPTS -Dpulsar.functions.process.container.log.dir=$PULSAR_LOG_DIR" +# instance +OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}" +OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}" #Change to PULSAR_HOME to support relative paths cd "$PULSAR_HOME" @@ -218,6 +270,9 @@ elif [ $COMMAND == "proxy" ]; then elif [ $COMMAND == "websocket" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-websocket.log"} exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.websocket.service.WebSocketServiceStarter $PULSAR_WEBSOCKET_CONF $@ +elif [ $COMMAND == "worker" ]; then + PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-function-worker.log"} + exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF $@ elif [ $COMMAND == "standalone" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-standalone.log"} exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarStandaloneStarter --config $PULSAR_STANDALONE_CONF $@ diff --git a/conf/function_worker.yml b/conf/function_worker.yml new file mode 100644 index 0000000000000..c2c40911e9e1c --- /dev/null +++ b/conf/function_worker.yml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +workerId: standalone +workerHostname: localhost +workerPort: 6750 +functionMetadataTopicName: metadata +functionMetadataSnapshotsTopicPath: snapshots +clusterCoordinationTopicName: coordinate +pulsarFunctionsNamespace: sample/standalone/functions +pulsarServiceUrl: pulsar://localhost:6650 +pulsarWebServiceUrl: http://localhost:8080 +numFunctionPackageReplicas: 1 +downloadDirectory: /tmp/pulsar_functions +metricsConfig: + metricsSinkClassName: org.apache.pulsar.functions.metrics.sink.PrometheusSink + metricsCollectionInterval: 30 + metricsSinkConfig: + path: /metrics + port: 9099 +#threadContainerFactory: +# threadGroupName: "Thread Function Container Group" +processContainerFactory: + logDirectory: + +schedulerClassName: "org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler" +functionAssignmentTopicName: "assignments" +failureCheckFreqMs: 30000 +rescheduleTimeoutMs: 60000 +initialBrokerReconnectMaxRetries: 60 diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml index 13b07adc92ee8..cd0ec880b6070 100644 --- a/conf/log4j2.yaml +++ b/conf/log4j2.yaml @@ -23,6 +23,21 @@ Configuration: monitorInterval: 30 name: pulsar + Properties: + Property: + - name: "pulsar.log.dir" + value: "logs" + - name: "pulsar.log.file" + value: "pulsar.log" + - name: "pulsar.log.appender" + value: "RoutingAppender" + - name: "pulsar.log.level" + value: "info" + - name: "pulsar.routing.appender.default" + value: "Console" + - name: "bk.log.level" + value: "info" + # Example: logger-filter script Scripts: ScriptFile: @@ -32,19 +47,19 @@ Configuration: charset: UTF-8 Appenders: - - # Console + + # Console Console: name: Console target: SYSTEM_OUT PatternLayout: Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - - # Rolling file appender configuration + + # Rolling file appender configuration RollingFile: name: RollingFile fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}" - filePattern: "/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz" + filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz" immediateFlush: false PatternLayout: Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" @@ -54,7 +69,7 @@ Configuration: modulate: true SizeBasedTriggeringPolicy: size: 1 GB - # Trigger every day at midnight that also scan + # Trigger every day at midnight that also scan # roll-over strategy that deletes older file CronTriggeringPolicy: schedule: "0 0 0 * * ?" @@ -67,16 +82,101 @@ Configuration: glob: "*/${sys:pulsar.log.file}*log.gz" IfLastModified: age: 30d - + + # Rolling file appender configuration for bk + RollingRandomAccessFile: + name: BkRollingFile + fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk" + filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz" + immediateFlush: true + PatternLayout: + Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" + Policies: + TimeBasedTriggeringPolicy: + interval: 1 + modulate: true + SizeBasedTriggeringPolicy: + size: 1 GB + # Trigger every day at midnight that also scan + # roll-over strategy that deletes older file + CronTriggeringPolicy: + schedule: "0 0 0 * * ?" + # Delete file older than 30days + DefaultRolloverStrategy: + Delete: + basePath: ${sys:pulsar.log.dir} + maxDepth: 2 + IfFileName: + glob: "*/${sys:pulsar.log.file}.bk*log.gz" + IfLastModified: + age: 30d + + # Routing + Routing: + name: RoutingAppender + Routes: + pattern: "$${ctx:function}" + Route: + - + Routing: + name: InstanceRoutingAppender + Routes: + pattern: "$${ctx:instance}" + Route: + - + RollingFile: + name: "Rolling-${ctx:function}" + fileName : "${sys:pulsar.log.dir}/functions/${ctx:function}/function.log" + filePattern : "${sys:pulsar.log.dir}/functions/${ctx:function}-%d{MM-dd-yyyy}-%i.log.gz" + PatternLayout: + Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance: %X{instance}] %logger{1} - %msg%n" + Policies: + TimeBasedTriggeringPolicy: + interval: 1 + modulate: true + SizeBasedTriggeringPolicy: + size: "20MB" + # Trigger every day at midnight that also scan + # roll-over strategy that deletes older file + CronTriggeringPolicy: + schedule: "0 0 0 * * ?" + # Delete file older than 30days + DefaultRolloverStrategy: + Delete: + basePath: ${sys:pulsar.log.dir} + maxDepth: 2 + IfFileName: + glob: "*/${sys:pulsar.log.file}*log.gz" + IfLastModified: + age: 30d + - ref: "${sys:pulsar.routing.appender.default}" + key: "${ctx:function}" + - ref: "${sys:pulsar.routing.appender.default}" + key: "${ctx:function}" Loggers: + + Logger: + name: org.apache.bookkeeper + level: "${sys:bk.log.level}" + additivity: false + AppenderRef: + - ref: BkRollingFile + + Logger: + name: org.apache.distributedlog + level: "${sys:bk.log.level}" + additivity: false + AppenderRef: + - ref: BkRollingFile # Default root logger configuration Root: level: info additivity: false AppenderRef: - ref: "${sys:pulsar.log.appender}" + - ref: "${sys:pulsar.log.appender}" + level: "${sys:pulsar.log.level}" # Logger to inject filter script # Logger: @@ -90,4 +190,4 @@ Configuration: # onMisMatch: DENY # ScriptRef: # ref: filter.js - \ No newline at end of file + diff --git a/pom.xml b/pom.xml index 8c7b160aa67c3..b6ac3b15e90c1 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,7 @@ flexible messaging model and an intuitive client API. pulsar-client-admin-shaded pulsar-client-tools pulsar-client-tools-shaded + pulsar-client-tools-test pulsar-websocket pulsar-proxy pulsar-discovery-service diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index f4744aa943502..abbc8313f8aec 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -119,12 +119,96 @@ test + + ${project.groupId} pulsar-functions-worker-shaded ${project.version} + + + io.grpc + grpc-all + + + com.google.protobuf + protobuf-lite + + + com.google.protobuf + protobuf-java + + + com.google.protobuf.nano + protobuf-javanano + + + com.google.protobuf + protobuf-java-util + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-protobuf-lite + + + io.grpc + grpc-protobuf-nano + + + org.apache.pulsar + pulsar-functions-utils + + + org.apache.pulsar + pulsar-functions-metrics + + + org.apache.pulsar + pulsar-functions-runtime + + + org.apache.pulsar + pulsar-functions-instance + + + org.apache.pulsar + pulsar-functions-worker + + + + + + net.jodah + typetools + + + + org.apache.pulsar + pulsar-functions-api + ${project.version} + + + + com.google.code.gson + gson + + org.glassfish.jersey.media + jersey-media-multipart + + + + org.apache.distributedlog + distributedlog-core-shaded + + + + org.eclipse.jetty jetty-server diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java index 40f8d59ca1ce6..b500e96b265f3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java @@ -34,6 +34,7 @@ import java.net.MalformedURLException; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Optional; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.proto.BookieServer; import org.apache.bookkeeper.replication.AutoRecoveryMain; @@ -42,6 +43,9 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.ServiceConfigurationUtils; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.apache.pulsar.functions.worker.WorkerService; import org.aspectj.weaver.loadtime.Agent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,6 +76,12 @@ private static class StarterArguments { @Parameter(names = {"-bc", "--bookie-conf"}, description = "Configuration file for Bookie") private String bookieConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/bookkeeper.conf"; + @Parameter(names = {"-rfw", "--run-function-worker"}, description = "Run function worker with Broker") + private boolean runFunctionWorker = false; + + @Parameter(names = {"-wc", "--function-worker-conf"}, description = "Configuration file for Function Worker") + private String fnWorkerConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/function_worker.yml"; + @Parameter(names = {"-h", "--help"}, description = "Show this help message") private boolean help = false; } @@ -103,6 +113,7 @@ private static class BrokerStarter { private final AutoRecoveryMain autoRecoveryMain; private final StatsProvider bookieStatsProvider; private final ServerConfiguration bookieConfig; + private final WorkerService functionWorkerService; BrokerStarter(String[] args) throws Exception{ StarterArguments starterArguments = new StarterArguments(); @@ -116,15 +127,40 @@ private static class BrokerStarter { System.exit(-1); } - // init broker config and pulsar service + // init broker config if (isBlank(starterArguments.brokerConfigFile)) { jcommander.usage(); throw new IllegalArgumentException("Need to specify a configuration file for broker"); } else { brokerConfig = loadConfig(starterArguments.brokerConfigFile); - pulsarService = new PulsarService(brokerConfig); } + // init function worker + if (starterArguments.runFunctionWorker) { + WorkerConfig workerConfig; + if (isBlank(starterArguments.fnWorkerConfigFile)) { + workerConfig = new WorkerConfig(); + } else { + workerConfig = WorkerConfig.load(starterArguments.fnWorkerConfigFile); + } + // worker talks to local broker + workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + brokerConfig.getBrokerServicePort()); + workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:" + brokerConfig.getWebServicePort()); + String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress( + brokerConfig.getAdvertisedAddress()); + workerConfig.setWorkerHostname(hostname); + workerConfig.setWorkerId( + "c-" + brokerConfig.getClusterName() + + "-fw-" + hostname + + "-" + workerConfig.getWorkerPort()); + functionWorkerService = new WorkerService(workerConfig); + } else { + functionWorkerService = null; + } + + // init pulsar service + pulsarService = new PulsarService(brokerConfig, Optional.ofNullable(functionWorkerService)); + // if no argument to run bookie in cmd line, read from pulsar config if (!argsContains(args, "-rb") && !argsContains(args, "--run-bookie")) { checkState(starterArguments.runBookie == false, @@ -189,6 +225,16 @@ public void start() throws Exception { pulsarService.start(); log.info("PulsarService started."); + + // after broker is started, start the function worker + if (null != functionWorkerService) { + try { + functionWorkerService.start(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw ie; + } + } } public void join() throws InterruptedException { @@ -203,6 +249,11 @@ public void join() throws InterruptedException { } public void shutdown() { + if (null != functionWorkerService) { + functionWorkerService.stop(); + log.info("Shut down function worker service successfully."); + } + pulsarService.getShutdownService().run(); log.info("Shut down broker service successfully."); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java index 314eacf6573d8..a9966b5c03952 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java @@ -23,6 +23,8 @@ import java.io.FileInputStream; import java.net.URL; +import java.nio.file.Paths; +import java.util.Optional; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.ServiceConfigurationUtils; @@ -31,6 +33,8 @@ import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.PropertyAdmin; +import org.apache.pulsar.functions.worker.WorkerConfig; +import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.aspectj.weaver.loadtime.Agent; import org.slf4j.Logger; @@ -48,6 +52,7 @@ public class PulsarStandaloneStarter { PulsarAdmin admin; LocalBookkeeperEnsemble bkEnsemble; ServiceConfiguration config; + WorkerService fnWorkerService; @Parameter(names = { "-c", "--config" }, description = "Configuration file path", required = true) private String configFile; @@ -76,6 +81,12 @@ public class PulsarStandaloneStarter { @Parameter(names = { "--only-broker" }, description = "Only start Pulsar broker service (no ZK, BK)") private boolean onlyBroker = false; + @Parameter(names = {"-rfw", "--run-function-worker"}, description = "Run function worker with Broker") + private boolean runFunctionWorker = false; + + @Parameter(names = {"-wc", "--function-worker-conf"}, description = "Configuration file for Function Worker") + private String fnWorkerConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/function_worker.yml"; + @Parameter(names = { "-a", "--advertised-address" }, description = "Standalone broker advertised address") private String advertisedAddress = null; @@ -127,6 +138,10 @@ public PulsarStandaloneStarter(String[] args) throws Exception { Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { + if (fnWorkerService != null) { + fnWorkerService.stop(); + } + if (broker != null) { broker.close(); } @@ -162,8 +177,29 @@ void start() throws Exception { // load aspectj-weaver agent for instrumentation AgentLoader.loadAgentClass(Agent.class.getName(), null); + // initialize the function worker + if (runFunctionWorker) { + WorkerConfig workerConfig; + if (isBlank(fnWorkerConfigFile)) { + workerConfig = new WorkerConfig(); + } else { + workerConfig = WorkerConfig.load(fnWorkerConfigFile); + } + // worker talks to local broker + workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePort()); + workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:" + config.getWebServicePort()); + String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress( + config.getAdvertisedAddress()); + workerConfig.setWorkerHostname(hostname); + workerConfig.setWorkerId( + "c-" + config.getClusterName() + + "-fw-" + hostname + + "-" + workerConfig.getWorkerPort()); + fnWorkerService = new WorkerService(workerConfig); + } + // Start Broker - broker = new PulsarService(config); + broker = new PulsarService(config, Optional.ofNullable(fnWorkerService)); broker.start(); // Create a sample namespace @@ -203,6 +239,10 @@ void start() throws Exception { log.info(e.getMessage()); } + if (null != fnWorkerService) { + fnWorkerService.start(); + } + log.debug("--- setup completed ---"); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 9b2343337b9ee..6bbc1e62c8be3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -24,6 +24,7 @@ import java.net.URL; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -62,6 +63,7 @@ import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.utils.PulsarBrokerVersionStringUtils; import org.apache.pulsar.websocket.WebSocketConsumerServlet; import org.apache.pulsar.websocket.WebSocketProducerServlet; @@ -121,6 +123,7 @@ public class PulsarService implements AutoCloseable { private final String brokerServiceUrl; private final String brokerServiceUrlTls; private final String brokerVersion; + private final Optional functionWorkerService; private final MessagingServiceShutdownHook shutdownService; @@ -136,6 +139,10 @@ public enum State { private final Condition isClosedCondition = mutex.newCondition(); public PulsarService(ServiceConfiguration config) { + this(config, Optional.empty()); + } + + public PulsarService(ServiceConfiguration config, Optional functionWorkerService) { // Validate correctness of configuration PulsarConfigurationLoader.isComplete(config); @@ -151,6 +158,7 @@ public PulsarService(ServiceConfiguration config) { this.shutdownService = new MessagingServiceShutdownHook(this); this.loadManagerExecutor = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-load-manager")); + this.functionWorkerService = functionWorkerService; } /** @@ -543,6 +551,10 @@ public NamespaceService getNamespaceService() { return this.nsservice; } + public WorkerService getWorkerService() { + return functionWorkerService.orElse(null); + } + /** * Get a reference of the current BrokerService instance associated with the current * PulsarService instance. diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java new file mode 100644 index 0000000000000..7080474ecbc16 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin.impl; + +import java.io.IOException; +import java.io.InputStream; +import java.util.function.Supplier; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.pulsar.broker.admin.AdminResource; +import org.apache.pulsar.functions.worker.WorkerService; +import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; + +public class FunctionsBase extends AdminResource implements Supplier { + + private final FunctionsImpl functions; + + public FunctionsBase() { + this.functions = new FunctionsImpl(this); + } + + @Override + public WorkerService get() { + return pulsar().getWorkerService(); + } + + @POST + @Path("/{tenant}/{namespace}/{functionName}") + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response registerFunction(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName, + final @FormDataParam("data") InputStream uploadedInputStream, + final @FormDataParam("data") FormDataContentDisposition fileDetail, + final @FormDataParam("functionConfig") String functionConfigJson) { + + return functions.registerFunction( + tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson); + + } + + @PUT + @Path("/{tenant}/{namespace}/{functionName}") + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response updateFunction(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName, + final @FormDataParam("data") InputStream uploadedInputStream, + final @FormDataParam("data") FormDataContentDisposition fileDetail, + final @FormDataParam("functionConfig") String functionConfigJson) { + + return functions.updateFunction( + tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson); + + } + + + @DELETE + @Path("/{tenant}/{namespace}/{functionName}") + public Response deregisterFunction(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName) { + return functions.deregisterFunction( + tenant, namespace, functionName); + } + + @GET + @Path("/{tenant}/{namespace}/{functionName}") + public Response getFunctionInfo(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName) + throws IOException { + return functions.getFunctionInfo( + tenant, namespace, functionName); + } + + @GET + @Path("/{tenant}/{namespace}/{functionName}/{instanceId}/status") + public Response getFunctionInstanceStatus(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName, + final @PathParam("instanceId") String instanceId) throws IOException { + return functions.getFunctionInstanceStatus( + tenant, namespace, functionName, instanceId); + } + + @GET + @Path("/{tenant}/{namespace}/{functionName}/status") + public Response getFunctionStatus(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName) throws IOException { + return functions.getFunctionStatus( + tenant, namespace, functionName); + } + + @GET + @Path("/{tenant}/{namespace}") + public Response listFunctions(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace) { + return functions.listFunctions( + tenant, namespace); + + } + + @GET + @Path("/cluster") + public Response getCluster() { + return functions.getCluster(); + } + + @GET + @Path("/assignments") + public Response getAssignments() { + return functions.getAssignments(); + } + +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Functions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Functions.java new file mode 100644 index 0000000000000..17407321991aa --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v1/Functions.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin.v1; + +import io.swagger.annotations.Api; +import javax.ws.rs.Path; +import org.apache.pulsar.broker.admin.impl.FunctionsBase; + +@Path("/functions") +@Api(value = "/functions", description = "Functions admin apis", tags = "functions") +public class Functions extends FunctionsBase { +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java new file mode 100644 index 0000000000000..cc68677654847 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Functions.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.admin.v2; + +import com.wordnik.swagger.annotations.Api; +import javax.ws.rs.Path; +import org.apache.pulsar.broker.admin.impl.FunctionsBase; + +@Path("/functions") +@Api(value = "/functions", description = "Functions admin apis", tags = "functions") +public class Functions extends FunctionsBase { +} diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java index 20ff44b922fa1..02c8b1a878d37 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java @@ -49,6 +49,7 @@ import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; +import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.servlet.ServletContainer; import org.slf4j.Logger; @@ -120,6 +121,7 @@ public void addRestResources(String basePath, String javaPackages, boolean requi ResourceConfig config = new ResourceConfig(); config.packages("jersey.config.server.provider.packages", javaPackages); config.register(provider); + config.register(MultiPartFeature.class); ServletHolder servletHolder = new ServletHolder(new ServletContainer(config)); servletHolder.setAsyncSupported(true); addServlet(basePath, servletHolder, requiresAuthentication, attributeMap); diff --git a/pulsar-client-tools-test/pom.xml b/pulsar-client-tools-test/pom.xml new file mode 100644 index 0000000000000..0aefc77024529 --- /dev/null +++ b/pulsar-client-tools-test/pom.xml @@ -0,0 +1,84 @@ + + + 4.0.0 + + org.apache.pulsar + pulsar + 2.0.0-incubating-SNAPSHOT + .. + + + pulsar-client-tools-test + Pulsar Client Tools Test + Pulsar Client Tools Test + + + + ${project.groupId} + pulsar-client-tools + ${project.version} + + + ${project.groupId} + managed-ledger + ${project.version} + test-jar + test + + + ${project.groupId} + pulsar-broker + ${project.version} + test-jar + test + + + ${project.groupId} + pulsar-broker + ${project.version} + test + + + ${project.groupId} + pulsar-zookeeper-utils + ${project.version} + test + + + org.mockito + mockito-core + test + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java similarity index 100% rename from pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java rename to pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/PulsarAdminToolTest.java diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/utils/IOUtilsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/utils/IOUtilsTest.java similarity index 100% rename from pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/utils/IOUtilsTest.java rename to pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/utils/IOUtilsTest.java diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitterTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitterTest.java similarity index 100% rename from pulsar-client-tools/src/test/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitterTest.java rename to pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/utils/NameValueParameterSplitterTest.java diff --git a/pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java similarity index 100% rename from pulsar-client-tools/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java rename to pulsar-client-tools-test/src/test/java/org/apache/pulsar/client/cli/PulsarClientToolTest.java diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 1d2454db790be..6ec1e801d51ab 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -19,89 +19,58 @@ --> - 4.0.0 - - org.apache.pulsar - pulsar - 2.0.0-incubating-SNAPSHOT - .. - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + org.apache.pulsar + pulsar + 2.0.0-incubating-SNAPSHOT + .. + - pulsar-client-tools - Pulsar Client Tools - Pulsar Client Tools + pulsar-client-tools + Pulsar Client Tools + Pulsar Client Tools - - - com.beust - jcommander - compile - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - junit - junit - - - - - commons-io - commons-io - - - ${project.groupId} - pulsar-common - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - org.asynchttpclient - async-http-client - - - org.apache.commons - commons-lang3 - - - ${project.groupId} - managed-ledger - ${project.version} - test-jar - test - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - ${project.groupId} - pulsar-broker - ${project.version} - test - - - ${project.groupId} - pulsar-zookeeper-utils - ${project.version} - test - - - org.mockito - mockito-core - test - - + + + com.beust + jcommander + compile + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + junit + junit + + + + + commons-io + commons-io + + + ${project.groupId} + pulsar-common + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + org.asynchttpclient + async-http-client + + + org.apache.commons + commons-lang3 + + diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index 39dd271dcfd02..aeb53ae3ab66a 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -57,7 +57,6 @@ ${project.version} - org.apache.bookkeeper stream-storage-java-client diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index dc236b6736341..88828dc354014 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -45,6 +45,11 @@ ${project.version} + + io.grpc + grpc-all + + diff --git a/pulsar-functions/worker-runner/pom.xml b/pulsar-functions/worker-runner/pom.xml index 3f79a2ebb235f..1e621d5cc6f58 100644 --- a/pulsar-functions/worker-runner/pom.xml +++ b/pulsar-functions/worker-runner/pom.xml @@ -56,10 +56,6 @@ com.google.protobuf protobuf-java-util - - org.apache.pulsar - pulsar-functions-proto - io.grpc grpc-protobuf @@ -72,6 +68,30 @@ io.grpc grpc-protobuf-nano + + org.apache.pulsar + pulsar-functions-proto + + + org.apache.pulsar + pulsar-functions-utils + + + org.apache.pulsar + pulsar-functions-metrics + + + org.apache.pulsar + pulsar-functions-runtime + + + org.apache.pulsar + pulsar-functions-instance + + + org.apache.pulsar + pulsar-functions-worker + diff --git a/pulsar-functions/worker-shaded/pom.xml b/pulsar-functions/worker-shaded/pom.xml index 2e079db32ed6e..670c8b709f91e 100644 --- a/pulsar-functions/worker-shaded/pom.xml +++ b/pulsar-functions/worker-shaded/pom.xml @@ -76,17 +76,25 @@ false + com.google.protobuf:protobuf-lite com.google.protobuf:protobuf-java com.google.protobuf.nano:protobuf-javanano + com.google.protobuf:protobuf-java-util + com.google.instrumentation:instrumentation-api + com.google.api.grpc:proto-google-common-protos - org.apache.pulsar:pulsar-functions-* + org.apache.pulsar:pulsar-functions-proto + org.apache.pulsar:pulsar-functions-utils + org.apache.pulsar:pulsar-functions-metrics + org.apache.pulsar:pulsar-functions-instance + org.apache.pulsar:pulsar-functions-runtime + org.apache.pulsar:pulsar-functions-worker - com.google.protobuf:protobuf-java-util - io.grpc:grpc-protobuf - io.grpc:grpc-protobuf-lite - io.grpc:grpc-protobuf-nano + io.grpc:* + + org.apache.bookkeeper:stream-storage-java-client diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java index 754c8afdb810d..4942b34782faf 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/MembershipManager.java @@ -160,7 +160,7 @@ public static WorkerInfo of (String workerId, String workerHostname, int port) { public static WorkerInfo parseFrom(String str) { String[] tokens = str.split(":"); if (tokens.length != 3) { - throw new IllegalArgumentException("Invalid string to parse WorkerInfo"); + throw new IllegalArgumentException("Invalid string to parse WorkerInfo : " + str); } String workerId = tokens[0]; diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Worker.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Worker.java index 55450e6c63da7..09f9fc0fbd35b 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Worker.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/Worker.java @@ -20,10 +20,7 @@ import com.google.common.util.concurrent.AbstractService; -import java.io.IOException; -import java.net.URI; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.functions.worker.rest.WorkerServer; @Slf4j @@ -51,13 +48,7 @@ protected void doStart() { } protected void doStartImpl() throws InterruptedException { - URI dlogUri; - try { - dlogUri = FunctionMetadataSetup.setupFunctionMetadata(workerConfig); - } catch (PulsarAdminException | IOException e) { - throw new RuntimeException(e); - } - workerService.start(dlogUri); + workerService.start(); WorkerServer server = new WorkerServer(workerService); this.serverThread = new Thread(server, server.getThreadName()); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java index 5793e97a40f71..732ab07c97715 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java @@ -20,12 +20,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; import java.net.URI; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.distributedlog.DistributedLogConfiguration; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.distributedlog.api.namespace.NamespaceBuilder; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.functions.worker.rest.FunctionApiResource; @@ -71,7 +73,15 @@ public static ServletContextHandler newServletContextHandler(String contextPath, return contextHandler; } - public void start(URI dlogUri) throws InterruptedException { + public void start() throws InterruptedException { + try { + start(FunctionMetadataSetup.setupFunctionMetadata(workerConfig)); + } catch (PulsarAdminException | IOException e) { + throw new RuntimeException(e); + } + } + + private void start(URI dlogUri) throws InterruptedException { log.info("Starting worker {}...", workerConfig.getWorkerId()); try { log.info("Worker Configs: {}", new ObjectMapper().writerWithDefaultPrettyPrinter() diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java index 7d8b05431c4d9..1c5c73998200e 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/FunctionApiResource.java @@ -18,24 +18,30 @@ */ package org.apache.pulsar.functions.worker.rest; +import java.util.function.Supplier; import javax.servlet.ServletContext; import javax.ws.rs.core.Context; import org.apache.pulsar.functions.worker.WorkerService; +import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl; -public class FunctionApiResource { +public class FunctionApiResource implements Supplier { public static final String ATTRIBUTE_FUNCTION_WORKER = "function-worker"; + protected final FunctionsImpl functions; private WorkerService workerService; - @Context protected ServletContext servletContext; - public synchronized WorkerService worker() { + public FunctionApiResource() { + this.functions = new FunctionsImpl(this); + } + + @Override + public synchronized WorkerService get() { if (this.workerService == null) { this.workerService = (WorkerService) servletContext.getAttribute(ATTRIBUTE_FUNCTION_WORKER); } return this.workerService; } - } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java new file mode 100644 index 0000000000000..4eda822ef2bcd --- /dev/null +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -0,0 +1,555 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.functions.worker.rest.api; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.gson.Gson; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.common.policies.data.ErrorData; +import org.apache.pulsar.functions.proto.Function; +import org.apache.pulsar.functions.proto.Function.FunctionConfig; +import org.apache.pulsar.functions.proto.Function.FunctionMetaData; +import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; +import org.apache.pulsar.functions.proto.InstanceCommunication; +import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus; +import org.apache.pulsar.functions.worker.FunctionMetaDataManager; +import org.apache.pulsar.functions.worker.FunctionRuntimeManager; +import org.apache.pulsar.functions.worker.MembershipManager; +import org.apache.pulsar.functions.worker.Utils; +import org.apache.pulsar.functions.worker.WorkerService; +import org.apache.pulsar.functions.worker.request.RequestResult; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; + +@Slf4j +public class FunctionsImpl { + + private final Supplier workerServiceSupplier; + + public FunctionsImpl(Supplier workerServiceSupplier) { + this.workerServiceSupplier = workerServiceSupplier; + } + + private WorkerService worker() { + try { + return checkNotNull(workerServiceSupplier.get()); + } catch (Throwable t) { + log.info("Failed to get worker service", t); + throw t; + } + } + + @POST + @Path("/{tenant}/{namespace}/{functionName}") + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response registerFunction(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName, + final @FormDataParam("data") InputStream uploadedInputStream, + final @FormDataParam("data") FormDataContentDisposition fileDetail, + final @FormDataParam("functionConfig") String functionConfigJson) { + FunctionConfig functionConfig; + // validate parameters + try { + functionConfig = validateUpdateRequestParams(tenant, namespace, functionName, + uploadedInputStream, fileDetail, functionConfigJson); + } catch (IllegalArgumentException e) { + log.error("Invalid register function request @ /{}/{}/{}", + tenant, namespace, functionName, e); + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getMessage())).build(); + } + + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + + if (functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { + log.error("Function {}/{}/{} already exists", tenant, namespace, functionName); + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(String.format("Function %s already exist", functionName))).build(); + } + + // function state + FunctionMetaData.Builder functionMetaDataBuilder = FunctionMetaData.newBuilder() + .setFunctionConfig(functionConfig) + .setCreateTime(System.currentTimeMillis()) + .setVersion(0); + + PackageLocationMetaData.Builder packageLocationMetaDataBuilder = PackageLocationMetaData.newBuilder() + .setPackagePath(String.format( + "%s/%s/%s/%s", + tenant, + namespace, + functionName, + Utils.getUniquePackageName(fileDetail.getFileName()))); + functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); + + return updateRequest(functionMetaDataBuilder.build(), uploadedInputStream); + } + + @PUT + @Path("/{tenant}/{namespace}/{functionName}") + @Consumes(MediaType.MULTIPART_FORM_DATA) + public Response updateFunction(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName, + final @FormDataParam("data") InputStream uploadedInputStream, + final @FormDataParam("data") FormDataContentDisposition fileDetail, + final @FormDataParam("functionConfig") String functionConfigJson) { + + FunctionConfig functionConfig; + // validate parameters + try { + functionConfig = validateUpdateRequestParams(tenant, namespace, functionName, + uploadedInputStream, fileDetail, functionConfigJson); + } catch (IllegalArgumentException e) { + log.error("Invalid update function request @ /{}/{}/{}", + tenant, namespace, functionName, e); + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getMessage())).build(); + } + + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + + if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); + } + + // function state + FunctionMetaData.Builder functionMetaDataBuilder = FunctionMetaData.newBuilder() + .setFunctionConfig(functionConfig) + .setCreateTime(System.currentTimeMillis()) + .setVersion(0); + + PackageLocationMetaData.Builder packageLocationMetaDataBuilder = PackageLocationMetaData.newBuilder() + .setPackagePath(String.format( + "%s/%s/%s/%s", + tenant, + namespace, + functionName, + Utils.getUniquePackageName(fileDetail.getFileName()))); + functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); + + return updateRequest(functionMetaDataBuilder.build(), uploadedInputStream); + } + + + @DELETE + @Path("/{tenant}/{namespace}/{functionName}") + public Response deregisterFunction(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName) { + + // validate parameters + try { + validateDeregisterRequestParams(tenant, namespace, functionName); + } catch (IllegalArgumentException e) { + log.error("Invalid deregister function request @ /{}/{}/{}", + tenant, namespace, functionName, e); + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getMessage())).build(); + } + + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { + log.error("Function to deregister does not exist @ /{}/{}/{}", + tenant, namespace, functionName); + return Response.status(Status.NOT_FOUND) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); + } + + CompletableFuture completableFuture + = functionMetaDataManager.deregisterFunction(tenant, namespace, functionName); + + RequestResult requestResult = null; + try { + requestResult = completableFuture.get(); + if (!requestResult.isSuccess()) { + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(requestResult.getMessage())) + .build(); + } + } catch (ExecutionException e) { + log.error("Execution Exception while deregistering function @ /{}/{}/{}", + tenant, namespace, functionName, e); + return Response.serverError() + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getCause().getMessage())) + .build(); + } catch (InterruptedException e) { + log.error("Interrupted Exception while deregistering function @ /{}/{}/{}", + tenant, namespace, functionName, e); + return Response.status(Status.REQUEST_TIMEOUT) + .type(MediaType.APPLICATION_JSON) + .build(); + } + + return Response.status(Status.OK).entity(requestResult.toJson()).build(); + } + + @GET + @Path("/{tenant}/{namespace}/{functionName}") + public Response getFunctionInfo(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName) + throws InvalidProtocolBufferException { + + // validate parameters + try { + validateGetFunctionRequestParams(tenant, namespace, functionName); + } catch (IllegalArgumentException e) { + log.error("Invalid getFunction request @ /{}/{}/{}", + tenant, namespace, functionName, e); + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getMessage())).build(); + } + + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { + log.error("Function in getFunction does not exist @ /{}/{}/{}", + tenant, namespace, functionName); + return Response.status(Status.NOT_FOUND) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); + } + + FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName); + String functionConfigJson = JsonFormat.printer().print(functionMetaData.getFunctionConfig()); + return Response.status(Status.OK).entity(functionConfigJson).build(); + } + + @GET + @Path("/{tenant}/{namespace}/{functionName}/{instanceId}/status") + public Response getFunctionInstanceStatus(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName, + final @PathParam("instanceId") String instanceId) throws IOException { + + // validate parameters + try { + validateGetFunctionInstanceRequestParams(tenant, namespace, functionName, instanceId); + } catch (IllegalArgumentException e) { + log.error("Invalid getFunctionStatus request @ /{}/{}/{}", + tenant, namespace, functionName, e); + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getMessage())).build(); + } + + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { + log.error("Function in getFunctionStatus does not exist @ /{}/{}/{}", + tenant, namespace, functionName); + return Response.status(Status.NOT_FOUND) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); + } + + FunctionRuntimeManager functionRuntimeManager = worker().getFunctionRuntimeManager(); + FunctionStatus functionStatus = null; + try { + functionStatus = functionRuntimeManager.getFunctionInstanceStatus( + tenant, namespace, functionName, Integer.parseInt(instanceId)); + } catch (Exception e) { + log.error("Got Exception Getting Status", e); + FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder(); + functionStatusBuilder.setRunning(false); + String functionConfigJson = JsonFormat.printer().print(functionStatusBuilder.build()); + return Response.status(Status.OK).entity(functionConfigJson).build(); + } + + String jsonResponse = JsonFormat.printer().print(functionStatus); + return Response.status(Status.OK).entity(jsonResponse).build(); + } + + @GET + @Path("/{tenant}/{namespace}/{functionName}/status") + public Response getFunctionStatus(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace, + final @PathParam("functionName") String functionName) throws IOException { + + // validate parameters + try { + validateGetFunctionRequestParams(tenant, namespace, functionName); + } catch (IllegalArgumentException e) { + log.error("Invalid getFunctionStatus request @ /{}/{}/{}", + tenant, namespace, functionName, e); + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getMessage())).build(); + } + + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { + log.error("Function in getFunctionStatus does not exist @ /{}/{}/{}", + tenant, namespace, functionName); + return Response.status(Status.NOT_FOUND) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); + } + + FunctionRuntimeManager functionRuntimeManager = worker().getFunctionRuntimeManager(); + InstanceCommunication.FunctionStatusList functionStatusList = null; + try { + functionStatusList = functionRuntimeManager.getAllFunctionStatus(tenant, namespace, functionName); + } catch (Exception e) { + log.error("Got Exception Getting Status", e); + FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder(); + functionStatusBuilder.setRunning(false); + String functionConfigJson = JsonFormat.printer().print(functionStatusBuilder.build()); + return Response.status(Status.OK).entity(functionConfigJson).build(); + } + + String jsonResponse = JsonFormat.printer().print(functionStatusList); + return Response.status(Status.OK).entity(jsonResponse).build(); + } + + @GET + @Path("/{tenant}/{namespace}") + public Response listFunctions(final @PathParam("tenant") String tenant, + final @PathParam("namespace") String namespace) { + + // validate parameters + try { + validateListFunctionRequestParams(tenant, namespace); + } catch (IllegalArgumentException e) { + log.error("Invalid listFunctions request @ /{}/{}", + tenant, namespace, e); + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getMessage())).build(); + } + + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + + Collection functionStateList = functionMetaDataManager.listFunctions(tenant, namespace); + + return Response.status(Status.OK).entity(new Gson().toJson(functionStateList.toArray())).build(); + } + + private Response updateRequest(FunctionMetaData functionMetaData, + InputStream uploadedInputStream) { + // Upload to bookkeeper + try { + log.info("Uploading function package to {}", functionMetaData.getPackageLocation()); + + Utils.uploadToBookeeper( + worker().getDlogNamespace(), + uploadedInputStream, + functionMetaData.getPackageLocation().getPackagePath()); + } catch (IOException e) { + log.error("Error uploading file {}", functionMetaData.getPackageLocation(), e); + return Response.serverError() + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getMessage())) + .build(); + } + + // Submit to FMT + FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); + + CompletableFuture completableFuture + = functionMetaDataManager.updateFunction(functionMetaData); + + RequestResult requestResult = null; + try { + requestResult = completableFuture.get(); + if (!requestResult.isSuccess()) { + return Response.status(Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(requestResult.getMessage())) + .build(); + } + } catch (ExecutionException e) { + return Response.serverError() + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getCause().getMessage())) + .build(); + } catch (InterruptedException e) { + return Response.status(Status.REQUEST_TIMEOUT) + .type(MediaType.APPLICATION_JSON) + .entity(new ErrorData(e.getCause().getMessage())) + .build(); + } + + return Response.status(Status.OK).build(); + } + + @GET + @Path("/cluster") + public Response getCluster() { + MembershipManager membershipManager = worker().getMembershipManager(); + List members = membershipManager.getCurrentMembership(); + return Response.status(Status.OK).entity(new Gson().toJson(members)).build(); + } + + @GET + @Path("/assignments") + public Response getAssignments() { + FunctionRuntimeManager functionRuntimeManager = worker().getFunctionRuntimeManager(); + Map> assignments = functionRuntimeManager.getCurrentAssignments(); + Map> ret = new HashMap<>(); + for (Map.Entry> entry : assignments.entrySet()) { + ret.put(entry.getKey(), entry.getValue().keySet()); + } + return Response.status(Status.OK).entity( + new Gson().toJson(ret)).build(); + } + + private void validateListFunctionRequestParams(String tenant, String namespace) throws IllegalArgumentException { + + if (tenant == null) { + throw new IllegalArgumentException("Tenant is not provided"); + } + if (namespace == null) { + throw new IllegalArgumentException("Namespace is not provided"); + } + } + + private void validateGetFunctionInstanceRequestParams(String tenant, + String namespace, + String functionName, + String instanceId) throws IllegalArgumentException { + validateGetFunctionRequestParams(tenant, namespace, functionName); + if (instanceId == null) { + throw new IllegalArgumentException("Function Instance Id is not provided"); + + } + } + + private void validateGetFunctionRequestParams(String tenant, + String namespace, + String functionName) throws IllegalArgumentException { + + if (tenant == null) { + throw new IllegalArgumentException("Tenant is not provided"); + } + if (namespace == null) { + throw new IllegalArgumentException("Namespace is not provided"); + } + if (functionName == null) { + throw new IllegalArgumentException("Function Name is not provided"); + } + } + + private void validateDeregisterRequestParams(String tenant, + String namespace, + String functionName) throws IllegalArgumentException { + + if (tenant == null) { + throw new IllegalArgumentException("Tenant is not provided"); + } + if (namespace == null) { + throw new IllegalArgumentException("Namespace is not provided"); + } + if (functionName == null) { + throw new IllegalArgumentException("Function Name is not provided"); + } + } + + private FunctionConfig validateUpdateRequestParams(String tenant, + String namespace, + String functionName, + InputStream uploadedInputStream, + FormDataContentDisposition fileDetail, + String functionConfigJson) throws IllegalArgumentException { + if (tenant == null) { + throw new IllegalArgumentException("Tenant is not provided"); + } + if (namespace == null) { + throw new IllegalArgumentException("Namespace is not provided"); + } + if (functionName == null) { + throw new IllegalArgumentException("Function Name is not provided"); + } + if (uploadedInputStream == null || fileDetail == null) { + throw new IllegalArgumentException("Function Package is not provided"); + } + if (functionConfigJson == null) { + throw new IllegalArgumentException("FunctionConfig is not provided"); + } + try { + FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); + JsonFormat.parser().merge(functionConfigJson, functionConfigBuilder); + FunctionConfig functionConfig = functionConfigBuilder.build(); + + List missingFields = new LinkedList<>(); + if (functionConfig.getTenant() == null || functionConfig.getTenant().isEmpty()) { + missingFields.add("Tenant"); + } + if (functionConfig.getNamespace() == null || functionConfig.getNamespace().isEmpty()) { + missingFields.add("Namespace"); + } + if (functionConfig.getName() == null || functionConfig.getName().isEmpty()) { + missingFields.add("Name"); + } + if (functionConfig.getClassName() == null || functionConfig.getClassName().isEmpty()) { + missingFields.add("ClassName"); + } + if (functionConfig.getInputsCount() == 0 && functionConfig.getCustomSerdeInputsCount() == 0) { + missingFields.add("Input"); + } + if (!missingFields.isEmpty()) { + String errorMessage = StringUtils.join(missingFields, ","); + throw new IllegalArgumentException(errorMessage + " is not provided"); + } + if (functionConfig.getParallelism() <= 0) { + throw new IllegalArgumentException("Parallelism needs to be set to a positive number"); + } + return functionConfig; + } catch (IllegalArgumentException ex) { + throw ex; + } catch (Exception ex) { + throw new IllegalArgumentException("Invalid FunctionConfig"); + } + } + +} diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java index 72271964a132b..88cd1a4449735 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java @@ -18,25 +18,8 @@ */ package org.apache.pulsar.functions.worker.rest.api.v2; -import com.google.gson.Gson; -import javax.ws.rs.core.Response.Status; - import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.common.policies.data.ErrorData; -import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.functions.proto.Function.FunctionConfig; -import org.apache.pulsar.functions.proto.Function.FunctionMetaData; -import org.apache.pulsar.functions.proto.Function.PackageLocationMetaData; -import org.apache.pulsar.functions.proto.InstanceCommunication; -import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus; -import org.apache.pulsar.functions.worker.FunctionMetaDataManager; -import org.apache.pulsar.functions.worker.FunctionRuntimeManager; -import org.apache.pulsar.functions.worker.MembershipManager; -import org.apache.pulsar.functions.worker.Utils; -import org.apache.pulsar.functions.worker.request.RequestResult; import org.apache.pulsar.functions.worker.rest.FunctionApiResource; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -52,13 +35,6 @@ import javax.ws.rs.core.Response; import java.io.IOException; import java.io.InputStream; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; @Slf4j @Path("/functions") @@ -74,44 +50,9 @@ public Response registerFunction(final @PathParam("tenant") String tenant, final @FormDataParam("data") FormDataContentDisposition fileDetail, final @FormDataParam("functionConfig") String functionConfigJson) { - FunctionConfig functionConfig; - // validate parameters - try { - functionConfig = validateUpdateRequestParams(tenant, namespace, functionName, - uploadedInputStream, fileDetail, functionConfigJson); - } catch (IllegalArgumentException e) { - log.error("Invalid register function request @ /{}/{}/{}", - tenant, namespace, functionName, e); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getMessage())).build(); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - - if (functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { - log.error("Function {}/{}/{} already exists", tenant, namespace, functionName); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(String.format("Function %s already exist", functionName))).build(); - } - - // function state - FunctionMetaData.Builder functionMetaDataBuilder = FunctionMetaData.newBuilder() - .setFunctionConfig(functionConfig) - .setCreateTime(System.currentTimeMillis()) - .setVersion(0); + return functions.registerFunction( + tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson); - PackageLocationMetaData.Builder packageLocationMetaDataBuilder = PackageLocationMetaData.newBuilder() - .setPackagePath(String.format( - "%s/%s/%s/%s", - tenant, - namespace, - functionName, - Utils.getUniquePackageName(fileDetail.getFileName()))); - functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); - - return updateRequest(functionMetaDataBuilder.build(), uploadedInputStream); } @PUT @@ -124,43 +65,9 @@ public Response updateFunction(final @PathParam("tenant") String tenant, final @FormDataParam("data") FormDataContentDisposition fileDetail, final @FormDataParam("functionConfig") String functionConfigJson) { - FunctionConfig functionConfig; - // validate parameters - try { - functionConfig = validateUpdateRequestParams(tenant, namespace, functionName, - uploadedInputStream, fileDetail, functionConfigJson); - } catch (IllegalArgumentException e) { - log.error("Invalid update function request @ /{}/{}/{}", - tenant, namespace, functionName, e); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getMessage())).build(); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - - if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); - } + return functions.updateFunction( + tenant, namespace, functionName, uploadedInputStream, fileDetail, functionConfigJson); - // function state - FunctionMetaData.Builder functionMetaDataBuilder = FunctionMetaData.newBuilder() - .setFunctionConfig(functionConfig) - .setCreateTime(System.currentTimeMillis()) - .setVersion(0); - - PackageLocationMetaData.Builder packageLocationMetaDataBuilder = PackageLocationMetaData.newBuilder() - .setPackagePath(String.format( - "%s/%s/%s/%s", - tenant, - namespace, - functionName, - Utils.getUniquePackageName(fileDetail.getFileName()))); - functionMetaDataBuilder.setPackageLocation(packageLocationMetaDataBuilder); - - return updateRequest(functionMetaDataBuilder.build(), uploadedInputStream); } @@ -169,55 +76,8 @@ public Response updateFunction(final @PathParam("tenant") String tenant, public Response deregisterFunction(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) { - - // validate parameters - try { - validateDeregisterRequestParams(tenant, namespace, functionName); - } catch (IllegalArgumentException e) { - log.error("Invalid deregister function request @ /{}/{}/{}", - tenant, namespace, functionName, e); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getMessage())).build(); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { - log.error("Function to deregister does not exist @ /{}/{}/{}", - tenant, namespace, functionName); - return Response.status(Status.NOT_FOUND) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); - } - - CompletableFuture completableFuture - = functionMetaDataManager.deregisterFunction(tenant, namespace, functionName); - - RequestResult requestResult = null; - try { - requestResult = completableFuture.get(); - if (!requestResult.isSuccess()) { - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(requestResult.getMessage())) - .build(); - } - } catch (ExecutionException e) { - log.error("Execution Exception while deregistering function @ /{}/{}/{}", - tenant, namespace, functionName, e); - return Response.serverError() - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getCause().getMessage())) - .build(); - } catch (InterruptedException e) { - log.error("Interrupted Exception while deregistering function @ /{}/{}/{}", - tenant, namespace, functionName, e); - return Response.status(Status.REQUEST_TIMEOUT) - .type(MediaType.APPLICATION_JSON) - .build(); - } - - return Response.status(Response.Status.OK).entity(requestResult.toJson()).build(); + return functions.deregisterFunction( + tenant, namespace, functionName); } @GET @@ -226,30 +86,8 @@ public Response getFunctionInfo(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) throws InvalidProtocolBufferException { - - // validate parameters - try { - validateGetFunctionRequestParams(tenant, namespace, functionName); - } catch (IllegalArgumentException e) { - log.error("Invalid getFunction request @ /{}/{}/{}", - tenant, namespace, functionName, e); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getMessage())).build(); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { - log.error("Function in getFunction does not exist @ /{}/{}/{}", - tenant, namespace, functionName); - return Response.status(Status.NOT_FOUND) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); - } - - FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName); - String functionConfigJson = JsonFormat.printer().print(functionMetaData.getFunctionConfig()); - return Response.status(Response.Status.OK).entity(functionConfigJson).build(); + return functions.getFunctionInfo( + tenant, namespace, functionName); } @GET @@ -258,42 +96,8 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName, final @PathParam("instanceId") String instanceId) throws IOException { - - // validate parameters - try { - validateGetFunctionInstanceRequestParams(tenant, namespace, functionName, instanceId); - } catch (IllegalArgumentException e) { - log.error("Invalid getFunctionStatus request @ /{}/{}/{}", - tenant, namespace, functionName, e); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getMessage())).build(); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { - log.error("Function in getFunctionStatus does not exist @ /{}/{}/{}", - tenant, namespace, functionName); - return Response.status(Status.NOT_FOUND) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); - } - - FunctionRuntimeManager functionRuntimeManager = worker().getFunctionRuntimeManager(); - InstanceCommunication.FunctionStatus functionStatus = null; - try { - functionStatus = functionRuntimeManager.getFunctionInstanceStatus( - tenant, namespace, functionName, Integer.parseInt(instanceId)); - } catch (Exception e) { - log.error("Got Exception Getting Status", e); - FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder(); - functionStatusBuilder.setRunning(false); - String functionConfigJson = JsonFormat.printer().print(functionStatusBuilder.build()); - return Response.status(Response.Status.OK).entity(functionConfigJson).build(); - } - - String jsonResponse = JsonFormat.printer().print(functionStatus); - return Response.status(Response.Status.OK).entity(jsonResponse).build(); + return functions.getFunctionInstanceStatus( + tenant, namespace, functionName, instanceId); } @GET @@ -301,241 +105,29 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena public Response getFunctionStatus(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) throws IOException { - - // validate parameters - try { - validateGetFunctionRequestParams(tenant, namespace, functionName); - } catch (IllegalArgumentException e) { - log.error("Invalid getFunctionStatus request @ /{}/{}/{}", - tenant, namespace, functionName, e); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getMessage())).build(); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - if (!functionMetaDataManager.containsFunction(tenant, namespace, functionName)) { - log.error("Function in getFunctionStatus does not exist @ /{}/{}/{}", - tenant, namespace, functionName); - return Response.status(Status.NOT_FOUND) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(String.format("Function %s doesn't exist", functionName))).build(); - } - - FunctionRuntimeManager functionRuntimeManager = worker().getFunctionRuntimeManager(); - InstanceCommunication.FunctionStatusList functionStatusList = null; - try { - functionStatusList = functionRuntimeManager.getAllFunctionStatus(tenant, namespace, functionName); - } catch (Exception e) { - log.error("Got Exception Getting Status", e); - FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder(); - functionStatusBuilder.setRunning(false); - String functionConfigJson = JsonFormat.printer().print(functionStatusBuilder.build()); - return Response.status(Response.Status.OK).entity(functionConfigJson).build(); - } - - String jsonResponse = JsonFormat.printer().print(functionStatusList); - return Response.status(Response.Status.OK).entity(jsonResponse).build(); + return functions.getFunctionStatus( + tenant, namespace, functionName); } @GET @Path("/{tenant}/{namespace}") public Response listFunctions(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace) { + return functions.listFunctions( + tenant, namespace); - // validate parameters - try { - validateListFunctionRequestParams(tenant, namespace); - } catch (IllegalArgumentException e) { - log.error("Invalid listFunctions request @ /{}/{}", - tenant, namespace, e); - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getMessage())).build(); - } - - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - - Collection functionStateList = functionMetaDataManager.listFunctions(tenant, namespace); - - return Response.status(Response.Status.OK).entity(new Gson().toJson(functionStateList.toArray())).build(); - } - - private Response updateRequest(FunctionMetaData functionMetaData, - InputStream uploadedInputStream) { - // Upload to bookkeeper - try { - log.info("Uploading function package to {}", functionMetaData.getPackageLocation()); - - Utils.uploadToBookeeper( - worker().getDlogNamespace(), - uploadedInputStream, - functionMetaData.getPackageLocation().getPackagePath()); - } catch (IOException e) { - log.error("Error uploading file {}", functionMetaData.getPackageLocation(), e); - return Response.serverError() - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getMessage())) - .build(); - } - - // Submit to FMT - FunctionMetaDataManager functionMetaDataManager = worker().getFunctionMetaDataManager(); - - CompletableFuture completableFuture - = functionMetaDataManager.updateFunction(functionMetaData); - - RequestResult requestResult = null; - try { - requestResult = completableFuture.get(); - if (!requestResult.isSuccess()) { - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(requestResult.getMessage())) - .build(); - } - } catch (ExecutionException e) { - return Response.serverError() - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getCause().getMessage())) - .build(); - } catch (InterruptedException e) { - return Response.status(Status.REQUEST_TIMEOUT) - .type(MediaType.APPLICATION_JSON) - .entity(new ErrorData(e.getCause().getMessage())) - .build(); - } - - return Response.status(Response.Status.OK).build(); } @GET @Path("/cluster") public Response getCluster() { - MembershipManager membershipManager = worker().getMembershipManager(); - List members = membershipManager.getCurrentMembership(); - return Response.status(Response.Status.OK).entity(new Gson().toJson(members)).build(); + return functions.getCluster(); } @GET @Path("/assignments") public Response getAssignments() { - FunctionRuntimeManager functionRuntimeManager = worker().getFunctionRuntimeManager(); - Map> assignments = functionRuntimeManager.getCurrentAssignments(); - Map> ret = new HashMap<>(); - for (Map.Entry> entry : assignments.entrySet()) { - ret.put(entry.getKey(), entry.getValue().keySet()); - } - return Response.status(Response.Status.OK).entity( - new Gson().toJson(ret)).build(); - } - - private void validateListFunctionRequestParams(String tenant, String namespace) throws IllegalArgumentException { - - if (tenant == null) { - throw new IllegalArgumentException("Tenant is not provided"); - } - if (namespace == null) { - throw new IllegalArgumentException("Namespace is not provided"); - } - } - - private void validateGetFunctionInstanceRequestParams(String tenant, - String namespace, - String functionName, - String instanceId) throws IllegalArgumentException { - validateGetFunctionRequestParams(tenant, namespace, functionName); - if (instanceId == null) { - throw new IllegalArgumentException("Function Instance Id is not provided"); - - } - } - - private void validateGetFunctionRequestParams(String tenant, - String namespace, - String functionName) throws IllegalArgumentException { - - if (tenant == null) { - throw new IllegalArgumentException("Tenant is not provided"); - } - if (namespace == null) { - throw new IllegalArgumentException("Namespace is not provided"); - } - if (functionName == null) { - throw new IllegalArgumentException("Function Name is not provided"); - } - } - - private void validateDeregisterRequestParams(String tenant, - String namespace, - String functionName) throws IllegalArgumentException { - - if (tenant == null) { - throw new IllegalArgumentException("Tenant is not provided"); - } - if (namespace == null) { - throw new IllegalArgumentException("Namespace is not provided"); - } - if (functionName == null) { - throw new IllegalArgumentException("Function Name is not provided"); - } - } - - private FunctionConfig validateUpdateRequestParams(String tenant, - String namespace, - String functionName, - InputStream uploadedInputStream, - FormDataContentDisposition fileDetail, - String functionConfigJson) throws IllegalArgumentException { - if (tenant == null) { - throw new IllegalArgumentException("Tenant is not provided"); - } - if (namespace == null) { - throw new IllegalArgumentException("Namespace is not provided"); - } - if (functionName == null) { - throw new IllegalArgumentException("Function Name is not provided"); - } - if (uploadedInputStream == null || fileDetail == null) { - throw new IllegalArgumentException("Function Package is not provided"); - } - if (functionConfigJson == null) { - throw new IllegalArgumentException("FunctionConfig is not provided"); - } - try { - FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - JsonFormat.parser().merge(functionConfigJson, functionConfigBuilder); - FunctionConfig functionConfig = functionConfigBuilder.build(); - - List missingFields = new LinkedList<>(); - if (functionConfig.getTenant() == null || functionConfig.getTenant().isEmpty()) { - missingFields.add("Tenant"); - } - if (functionConfig.getNamespace() == null || functionConfig.getNamespace().isEmpty()) { - missingFields.add("Namespace"); - } - if (functionConfig.getName() == null || functionConfig.getName().isEmpty()) { - missingFields.add("Name"); - } - if (functionConfig.getClassName() == null || functionConfig.getClassName().isEmpty()) { - missingFields.add("ClassName"); - } - if (functionConfig.getInputsCount() == 0 && functionConfig.getCustomSerdeInputsCount() == 0) { - missingFields.add("Input"); - } - if (!missingFields.isEmpty()) { - String errorMessage = StringUtils.join(missingFields, ","); - throw new IllegalArgumentException(errorMessage + " is not provided"); - } - if (functionConfig.getParallelism() <= 0) { - throw new IllegalArgumentException("Parallelism needs to be set to a positive number"); - } - return functionConfig; - } catch (IllegalArgumentException ex) { - throw ex; - } catch (Exception ex) { - throw new IllegalArgumentException("Invalid FunctionConfig"); - } + return functions.getAssignments(); } } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index fe7594023728e..8bce7ffbfc78f 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -21,7 +21,6 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -57,6 +56,7 @@ import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; import org.apache.pulsar.functions.worker.request.RequestResult; +import org.apache.pulsar.functions.worker.rest.api.FunctionsImpl; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -98,13 +98,12 @@ public String process(String input, Context context) throws Exception { private WorkerService mockedWorkerService; private FunctionMetaDataManager mockedManager; private Namespace mockedNamespace; - private FunctionApiV2Resource resource; + private FunctionsImpl resource; private InputStream mockedInputStream; private FormDataContentDisposition mockedFormData; @BeforeMethod public void setup() { - this.resource = spy(new FunctionApiV2Resource()); this.mockedManager = mock(FunctionMetaDataManager.class); this.mockedInputStream = mock(InputStream.class); this.mockedFormData = mock(FormDataContentDisposition.class); @@ -124,7 +123,7 @@ public void setup() { .setPulsarServiceUrl("pulsar://localhost:6650/"); when(mockedWorkerService.getWorkerConfig()).thenReturn(workerConfig); - doReturn(mockedWorkerService).when(resource).worker(); + this.resource = spy(new FunctionsImpl(() -> mockedWorkerService)); } // From f30a7c2fe526e0ccb3d210ab6dc702513f2339cb Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Sun, 4 Mar 2018 23:06:56 -0800 Subject: [PATCH 09/14] rename 'function worker' to 'functions worker' --- bin/pulsar | 10 +++--- conf/broker.conf | 6 ++++ ...nction_worker.yml => functions_worker.yml} | 0 .../pulsar/broker/ServiceConfiguration.java | 13 ++++++++ .../apache/pulsar/PulsarBrokerStarter.java | 32 +++++++++---------- .../pulsar/PulsarStandaloneStarter.java | 12 +++---- 6 files changed, 46 insertions(+), 27 deletions(-) rename conf/{function_worker.yml => functions_worker.yml} (100%) diff --git a/bin/pulsar b/bin/pulsar index 0f12c302990f6..cafc3591b772a 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -33,7 +33,7 @@ DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml # functions related variables FUNCTIONS_HOME=$PULSAR_HOME/pulsar-functions -DEFAULT_WORKER_CONF=$PULSAR_HOME/conf/function_worker.yml +DEFAULT_WORKER_CONF=$PULSAR_HOME/conf/functions_worker.yml DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"} DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py @@ -110,7 +110,7 @@ where command is one of: discovery Run a discovery server proxy Run a pulsar proxy websocket Run a web socket proxy server - worker Run a function worker server + functions-worker Run a functions worker server standalone Run a broker server with local bookies and local zookeeper compact-topic Run compaction against a topic @@ -130,7 +130,7 @@ Environment variables: PULSAR_DISCOVERY_CONF Configuration file for discovery service (default: $DEFAULT_DISCOVERY_CONF) PULSAR_WEBSOCKET_CONF Configuration file for websocket proxy (default: $DEFAULT_WEBSOCKET_CONF) PULSAR_PROXY_CONF Configuration file for Pulsar proxy (default: $DEFAULT_PROXY_CONF) - PULSAR_WORKER_CONF Configuration file for function worker (default: $DEFAULT_WORKER_CONF) + PULSAR_WORKER_CONF Configuration file for functions worker (default: $DEFAULT_WORKER_CONF) PULSAR_STANDALONE_CONF Configuration file for standalone (default: $DEFAULT_STANDALONE_CONF) PULSAR_EXTRA_OPTS Extra options to be passed to the jvm PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath @@ -270,8 +270,8 @@ elif [ $COMMAND == "proxy" ]; then elif [ $COMMAND == "websocket" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-websocket.log"} exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.websocket.service.WebSocketServiceStarter $PULSAR_WEBSOCKET_CONF $@ -elif [ $COMMAND == "worker" ]; then - PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-function-worker.log"} +elif [ $COMMAND == "functions-worker" ]; then + PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-functions-worker.log"} exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF $@ elif [ $COMMAND == "standalone" ]; then PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-standalone.log"} diff --git a/conf/broker.conf b/conf/broker.conf index 93489e5cdd50f..55e6f19ec3868 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -440,3 +440,9 @@ webSocketConnectionsPerBroker=8 # Enable topic level metrics exposeTopicLevelMetricsInPrometheus=true + +### --- Functions --- ### + +# Enable Functions Worker Service in Broker +functionsWorkerEnabled=false + diff --git a/conf/function_worker.yml b/conf/functions_worker.yml similarity index 100% rename from conf/function_worker.yml rename to conf/functions_worker.yml diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 8aa2f821ffb42..bd4797023417d 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -425,6 +425,9 @@ public class ServiceConfiguration implements PulsarConfiguration { // If true, export topic level metrics otherwise namespace level private boolean exposeTopicLevelMetricsInPrometheus = true; + /**** --- Functions --- ****/ + private boolean functionsWorkerEnabled = false; + public String getZookeeperServers() { return zookeeperServers; } @@ -1470,4 +1473,14 @@ public Set getTlsCiphers() { public void setTlsCiphers(Set tlsCiphers) { this.tlsCiphers = tlsCiphers; } + + /**** --- Function ---- ****/ + + public void setFunctionsWorkerEnabled(boolean enabled) { + this.functionsWorkerEnabled = enabled; + } + + public boolean isFunctionsWorkerEnabled() { + return functionsWorkerEnabled; + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java index b500e96b265f3..42d90e2718373 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java @@ -76,11 +76,11 @@ private static class StarterArguments { @Parameter(names = {"-bc", "--bookie-conf"}, description = "Configuration file for Bookie") private String bookieConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/bookkeeper.conf"; - @Parameter(names = {"-rfw", "--run-function-worker"}, description = "Run function worker with Broker") - private boolean runFunctionWorker = false; + @Parameter(names = {"-rfw", "--run-functions-worker"}, description = "Run functions worker with Broker") + private boolean runFunctionsWorker = false; - @Parameter(names = {"-wc", "--function-worker-conf"}, description = "Configuration file for Function Worker") - private String fnWorkerConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/function_worker.yml"; + @Parameter(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker") + private String fnWorkerConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/functions_worker.yml"; @Parameter(names = {"-h", "--help"}, description = "Show this help message") private boolean help = false; @@ -113,7 +113,7 @@ private static class BrokerStarter { private final AutoRecoveryMain autoRecoveryMain; private final StatsProvider bookieStatsProvider; private final ServerConfiguration bookieConfig; - private final WorkerService functionWorkerService; + private final WorkerService functionsWorkerService; BrokerStarter(String[] args) throws Exception{ StarterArguments starterArguments = new StarterArguments(); @@ -135,8 +135,8 @@ private static class BrokerStarter { brokerConfig = loadConfig(starterArguments.brokerConfigFile); } - // init function worker - if (starterArguments.runFunctionWorker) { + // init functions worker + if (starterArguments.runFunctionsWorker || brokerConfig.isFunctionsWorkerEnabled()) { WorkerConfig workerConfig; if (isBlank(starterArguments.fnWorkerConfigFile)) { workerConfig = new WorkerConfig(); @@ -153,13 +153,13 @@ private static class BrokerStarter { "c-" + brokerConfig.getClusterName() + "-fw-" + hostname + "-" + workerConfig.getWorkerPort()); - functionWorkerService = new WorkerService(workerConfig); + functionsWorkerService = new WorkerService(workerConfig); } else { - functionWorkerService = null; + functionsWorkerService = null; } // init pulsar service - pulsarService = new PulsarService(brokerConfig, Optional.ofNullable(functionWorkerService)); + pulsarService = new PulsarService(brokerConfig, Optional.ofNullable(functionsWorkerService)); // if no argument to run bookie in cmd line, read from pulsar config if (!argsContains(args, "-rb") && !argsContains(args, "--run-bookie")) { @@ -226,10 +226,10 @@ public void start() throws Exception { pulsarService.start(); log.info("PulsarService started."); - // after broker is started, start the function worker - if (null != functionWorkerService) { + // after broker is started, start the functions worker + if (null != functionsWorkerService) { try { - functionWorkerService.start(); + functionsWorkerService.start(); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw ie; @@ -249,9 +249,9 @@ public void join() throws InterruptedException { } public void shutdown() { - if (null != functionWorkerService) { - functionWorkerService.stop(); - log.info("Shut down function worker service successfully."); + if (null != functionsWorkerService) { + functionsWorkerService.stop(); + log.info("Shut down functions worker service successfully."); } pulsarService.getShutdownService().run(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java index a9966b5c03952..6c8dc14dccc59 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandaloneStarter.java @@ -81,11 +81,11 @@ public class PulsarStandaloneStarter { @Parameter(names = { "--only-broker" }, description = "Only start Pulsar broker service (no ZK, BK)") private boolean onlyBroker = false; - @Parameter(names = {"-rfw", "--run-function-worker"}, description = "Run function worker with Broker") - private boolean runFunctionWorker = false; + @Parameter(names = {"-nfw", "--no-functions-worker"}, description = "Run functions worker with Broker") + private boolean noFunctionsWorker = false; - @Parameter(names = {"-wc", "--function-worker-conf"}, description = "Configuration file for Function Worker") - private String fnWorkerConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/function_worker.yml"; + @Parameter(names = {"-fwc", "--functions-worker-conf"}, description = "Configuration file for Functions Worker") + private String fnWorkerConfigFile = Paths.get("").toAbsolutePath().normalize().toString() + "/conf/functions_worker.yml"; @Parameter(names = { "-a", "--advertised-address" }, description = "Standalone broker advertised address") private String advertisedAddress = null; @@ -177,8 +177,8 @@ void start() throws Exception { // load aspectj-weaver agent for instrumentation AgentLoader.loadAgentClass(Agent.class.getName(), null); - // initialize the function worker - if (runFunctionWorker) { + // initialize the functions worker + if (!noFunctionsWorker) { WorkerConfig workerConfig; if (isBlank(fnWorkerConfigFile)) { workerConfig = new WorkerConfig(); From ce2082332fc55258ed67c6164522b7f15c656597 Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Mon, 5 Mar 2018 11:11:23 -0800 Subject: [PATCH 10/14] add "--no-functions-worker" for pulsar-client-cpp tests --- pulsar-client-cpp/run-unit-tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-client-cpp/run-unit-tests.sh b/pulsar-client-cpp/run-unit-tests.sh index f91e6ffb5c26e..9f8eb5fc88574 100755 --- a/pulsar-client-cpp/run-unit-tests.sh +++ b/pulsar-client-cpp/run-unit-tests.sh @@ -25,10 +25,11 @@ rm -rf ./pulsar-dist mkdir pulsar-dist tar xfz ../all/target/apache-pulsar*bin.tar.gz -C pulsar-dist --strip-components 1 -PULSAR_STANDALONE_CONF=$PWD/test-conf/standalone.conf pulsar-dist/bin/pulsar standalone > broker.log & +PULSAR_STANDALONE_CONF=$PWD/test-conf/standalone.conf pulsar-dist/bin/pulsar standalone --no-functions-worker > broker.log & standalone_pid=$!; PULSAR_STANDALONE_CONF=$PWD/test-conf/standalone-ssl.conf pulsar-dist/bin/pulsar standalone \ + --no-functions-worker \ --zookeeper-port 2191 --bookkeeper-port 3191 \ --zookeeper-dir data2/standalone/zookeeper --bookkeeper-dir \ data2/standalone/bookkeeper > broker-tls.log & From c5a7779a97187a7af9be2fce08296291db9e1509 Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Mon, 5 Mar 2018 00:29:24 -0800 Subject: [PATCH 11/14] Integrate function cli into pulsar-admin cli - rename `pulsar-client-tools-shaded` to `pulsar-client-admin-shaded-for-functions`, because this module is used by functions only to avoid protobuf conflicts - move protobuf3 references to Utils, so it won't be referenced out side of pulsar-functions - integrate function cli into pulsar-admin cli --- bin/pulsar-admin | 53 ++++++++-- {pulsar-functions/conf => conf}/example.yml | 0 pom.xml | 2 +- .../pom.xml | 13 +-- .../pulsar/admin/cli/CmdFunctionsTest.java | 9 +- .../test/resources/test_function_config.yml | 0 pulsar-client-tools/pom.xml | 85 ++++++++++++++++ .../apache/pulsar/admin/cli/CmdFunctions.java | 14 +-- .../pulsar/admin/cli/PulsarAdminTool.java | 35 ++++++- .../apache/pulsar/client/admin/Functions.java | 0 .../admin/PulsarAdminWithFunctions.java | 10 +- .../client/admin/internal/FunctionsImpl.java | 16 +-- pulsar-functions/cli/pom.xml | 59 ----------- .../pulsar/admin/cli/FunctionsTool.java | 99 ------------------- .../apache/pulsar/admin/cli/package-info.java | 24 ----- .../cli/src/test/resources/log4j2.yml | 37 ------- pulsar-functions/conf/example_logging.yml | 27 ----- pulsar-functions/dist/pom.xml | 6 -- .../functions/metrics/sink/FileSink.java | 4 +- pulsar-functions/pom.xml | 1 - pulsar-functions/utils/pom.xml | 2 +- .../functions/utils/FunctionConfigUtils.java | 3 +- .../apache/pulsar/functions/utils/Utils.java | 12 +++ pulsar-functions/worker-shaded/pom.xml | 4 +- .../worker/FunctionRuntimeManager.java | 7 +- .../worker/rest/api/FunctionsImpl.java | 16 ++- .../rest/api/v2/FunctionApiV2Resource.java | 3 +- .../api/v2/FunctionApiV2ResourceTest.java | 70 ++++++------- 28 files changed, 259 insertions(+), 352 deletions(-) rename {pulsar-functions/conf => conf}/example.yml (100%) rename {pulsar-client-tools-shaded => pulsar-client-admin-shaded-for-functions}/pom.xml (94%) rename {pulsar-functions/cli => pulsar-client-tools-test}/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java (98%) rename {pulsar-functions/cli => pulsar-client-tools-test}/src/test/resources/test_function_config.yml (100%) rename {pulsar-functions/cli => pulsar-client-tools}/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java (98%) rename {pulsar-functions/cli => pulsar-client-tools}/src/main/java/org/apache/pulsar/client/admin/Functions.java (100%) rename pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/PulsarFunctionsAdmin.java => pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/PulsarAdminWithFunctions.java (88%) rename {pulsar-functions/cli => pulsar-client-tools}/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java (91%) delete mode 100644 pulsar-functions/cli/pom.xml delete mode 100644 pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/FunctionsTool.java delete mode 100644 pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/package-info.java delete mode 100644 pulsar-functions/cli/src/test/resources/log4j2.yml delete mode 100644 pulsar-functions/conf/example_logging.yml diff --git a/bin/pulsar-admin b/bin/pulsar-admin index 8de438a785f70..2e49225622c12 100755 --- a/bin/pulsar-admin +++ b/bin/pulsar-admin @@ -24,6 +24,13 @@ PULSAR_HOME=`cd $BINDIR/..;pwd` DEFAULT_CLIENT_CONF=$PULSAR_HOME/conf/client.conf DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml +# functions related variables +FUNCTIONS_HOME=$PULSAR_HOME/pulsar-functions +DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar +JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"} +DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py +PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"} + if [ -f "$PULSAR_HOME/conf/pulsar_tools_env.sh" ] then . "$PULSAR_HOME/conf/pulsar_tools_env.sh" @@ -41,7 +48,7 @@ else fi # exclude tests jar -RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` +RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` if [ $? == 0 ]; then PULSAR_JAR=$RELEASE_JAR fi @@ -61,20 +68,20 @@ add_maven_deps_to_classpath() { if [ "$MAVEN_HOME" != "" ]; then MVN=${MAVEN_HOME}/bin/mvn fi - + # Need to generate classpath from maven pom. This is costly so generate it # and cache it. Save the file into our target dir so a mvn clean will get # clean it up and force us create a new one. f="${PULSAR_HOME}/all/target/classpath.txt" if [ ! -f "${f}" ] then - ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + ${MVN} -f "${PULSAR_HOME}/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null fi PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"` } if [ -d "$PULSAR_HOME/lib" ]; then - PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*" + PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*" else add_maven_deps_to_classpath fi @@ -97,11 +104,45 @@ OPTS="$OPTS $PULSAR_EXTRA_OPTS" # log directory & file PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} -PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"Console"} +PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} +PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} +PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} #Configure log configuration system properties -OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" +OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" +OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" +OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT" + +# find the java instance location +if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then + # didn't find a released jar, then search the built jar + BUILT_JAVA_INSTANCE_JAR="${FUNCTIONS_HOME}/runtime/target/java-instance.jar" + if [ -f "${BUILT_JAVA_INSTANCE_JAR}" ]; then + JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR} + else + echo "\nCouldn't find pulsar java instance jar."; + echo "Make sure you've run 'mvn package'\n"; + exit 1; + fi +fi + +# find the python instance location +if [ ! -f "${PY_INSTANCE_FILE}" ]; then + # didn't find a released python instance, then search the built python instance + BUILT_PY_INSTANCE_FILE="${FUNCTIONS_HOME}/runtime/target/python-instance/python_instance_main.py" + if [ -f "${BUILT_PY_INSTANCE_FILE}" ]; then + PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE} + else + echo "\nCouldn't find pulsar python instance."; + echo "Make sure you've run 'mvn package'\n"; + exit 1; + fi +fi + +# functions +OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}" +OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}" #Change to PULSAR_HOME to support relative paths cd "$PULSAR_HOME" diff --git a/pulsar-functions/conf/example.yml b/conf/example.yml similarity index 100% rename from pulsar-functions/conf/example.yml rename to conf/example.yml diff --git a/pom.xml b/pom.xml index b6ac3b15e90c1..742512d776b82 100644 --- a/pom.xml +++ b/pom.xml @@ -83,8 +83,8 @@ flexible messaging model and an intuitive client API. pulsar-client-shaded pulsar-client-admin pulsar-client-admin-shaded + pulsar-client-admin-shaded-for-functions pulsar-client-tools - pulsar-client-tools-shaded pulsar-client-tools-test pulsar-websocket pulsar-proxy diff --git a/pulsar-client-tools-shaded/pom.xml b/pulsar-client-admin-shaded-for-functions/pom.xml similarity index 94% rename from pulsar-client-tools-shaded/pom.xml rename to pulsar-client-admin-shaded-for-functions/pom.xml index 104ff42f78ed0..5953b753dc137 100644 --- a/pulsar-client-tools-shaded/pom.xml +++ b/pulsar-client-admin-shaded-for-functions/pom.xml @@ -29,13 +29,13 @@ .. - pulsar-client-tools-shaded - Pulsar Client Admin Shaded + pulsar-client-admin-shaded-for-functions + Pulsar Client Admin Shaded (for functions) ${project.groupId} - pulsar-client-tools + pulsar-client-admin-original ${project.parent.version} @@ -63,7 +63,6 @@ org.apache.bookkeeper:circe-checksum org.apache.pulsar:pulsar-client-original org.apache.pulsar:pulsar-client-admin-original - org.apache.pulsar:pulsar-client-tools org.apache.commons:commons-lang3 commons-codec:commons-codec @@ -119,12 +118,6 @@ ** - - org.apache.pulsar:pulsar-client-tools - - ** - - commons-logging:commons-logging diff --git a/pulsar-functions/cli/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java similarity index 98% rename from pulsar-functions/cli/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java rename to pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index 5f390f82ced34..daf29c52af59c 100644 --- a/pulsar-functions/cli/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -52,8 +52,7 @@ import org.apache.pulsar.admin.cli.CmdFunctions.LocalRunner; import org.apache.pulsar.admin.cli.CmdFunctions.UpdateFunction; import org.apache.pulsar.client.admin.Functions; -import org.apache.pulsar.client.admin.PulsarFunctionsAdmin; -import org.apache.pulsar.client.api.ClientConfiguration; +import org.apache.pulsar.client.admin.PulsarAdminWithFunctions; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.functions.api.Context; import org.apache.pulsar.functions.api.PulsarFunction; @@ -82,7 +81,7 @@ public IObjectFactory getObjectFactory() { private static final String TEST_NAME = "test_name"; - private PulsarFunctionsAdmin admin; + private PulsarAdminWithFunctions admin; private Functions functions; private CmdFunctions cmd; @@ -101,7 +100,7 @@ private String generateCustomSerdeInputs(String topic, String serde) { @BeforeMethod public void setup() throws Exception { - this.admin = mock(PulsarFunctionsAdmin.class); + this.admin = mock(PulsarAdminWithFunctions.class); this.functions = mock(Functions.class); when(admin.functions()).thenReturn(functions); when(admin.getServiceUrl()).thenReturn(URI.create("http://localhost:1234").toURL()); diff --git a/pulsar-functions/cli/src/test/resources/test_function_config.yml b/pulsar-client-tools-test/src/test/resources/test_function_config.yml similarity index 100% rename from pulsar-functions/cli/src/test/resources/test_function_config.yml rename to pulsar-client-tools-test/src/test/resources/test_function_config.yml diff --git a/pulsar-client-tools/pom.xml b/pulsar-client-tools/pom.xml index 6ec1e801d51ab..693eca1e2ff5d 100644 --- a/pulsar-client-tools/pom.xml +++ b/pulsar-client-tools/pom.xml @@ -71,6 +71,91 @@ org.apache.commons commons-lang3 + + + + + ${project.groupId} + pulsar-functions-worker-shaded + ${project.version} + + + io.grpc + grpc-all + + + com.google.protobuf + protobuf-lite + + + com.google.protobuf + protobuf-java + + + com.google.protobuf.nano + protobuf-javanano + + + com.google.protobuf + protobuf-java-util + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-protobuf-lite + + + io.grpc + grpc-protobuf-nano + + + org.apache.pulsar + pulsar-functions-utils + + + org.apache.pulsar + pulsar-functions-metrics + + + org.apache.pulsar + pulsar-functions-runtime + + + org.apache.pulsar + pulsar-functions-instance + + + org.apache.pulsar + pulsar-functions-worker + + + + + + net.jodah + typetools + + + + org.apache.pulsar + pulsar-functions-api + ${project.version} + + + + com.google.code.gson + gson + + + + org.glassfish.jersey.media + jersey-media-multipart + + + diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java similarity index 98% rename from pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 97fbf7bc3a552..bb2228c5731fa 100644 --- a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -30,7 +30,6 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonParser; import com.google.gson.reflect.TypeToken; -import com.google.protobuf.util.JsonFormat; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; @@ -45,7 +44,7 @@ import org.apache.bookkeeper.clients.config.StorageClientSettings; import org.apache.bookkeeper.clients.utils.NetUtils; import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarFunctionsAdmin; +import org.apache.pulsar.client.admin.PulsarAdminWithFunctions; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.functions.api.PulsarFunction; import org.apache.pulsar.functions.api.utils.DefaultSerDe; @@ -64,12 +63,13 @@ import java.util.Map; import java.util.UUID; import java.util.function.Function; +import org.apache.pulsar.functions.utils.Utils; @Slf4j @Parameters(commandDescription = "Operations about functions") public class CmdFunctions extends CmdBase { - private final PulsarFunctionsAdmin fnAdmin; + private final PulsarAdminWithFunctions fnAdmin; private final LocalRunner localRunner; private final CreateFunction creater; private final DeleteFunction deleter; @@ -451,7 +451,7 @@ void runCmd() throws Exception { throw new RuntimeException("Missing arguments"); } - String serviceUrl = ((PulsarFunctionsAdmin) admin).getClientConf().getServiceUrl(); + String serviceUrl = ((PulsarAdminWithFunctions) admin).getClientConf().getServiceUrl(); if (brokerServiceUrl != null) { serviceUrl = brokerServiceUrl; } @@ -511,7 +511,7 @@ void runCmd() throws Exception { class GetFunction extends FunctionCommand { @Override void runCmd() throws Exception { - String json = JsonFormat.printer().print(fnAdmin.functions().getFunction(tenant, namespace, functionName)); + String json = Utils.printJson(fnAdmin.functions().getFunction(tenant, namespace, functionName)); Gson gson = new GsonBuilder().setPrettyPrinting().create(); System.out.println(gson.toJson(new JsonParser().parse(json))); } @@ -521,7 +521,7 @@ void runCmd() throws Exception { class GetFunctionStatus extends FunctionCommand { @Override void runCmd() throws Exception { - String json = JsonFormat.printer().print(fnAdmin.functions().getFunctionStatus(tenant, namespace, functionName)); + String json = Utils.printJson(fnAdmin.functions().getFunctionStatus(tenant, namespace, functionName)); Gson gson = new GsonBuilder().setPrettyPrinting().create(); System.out.println(gson.toJson(new JsonParser().parse(json))); } @@ -616,7 +616,7 @@ void runCmd() throws Exception { public CmdFunctions(PulsarAdmin admin) { super("functions", admin); - this.fnAdmin = (PulsarFunctionsAdmin) admin; + this.fnAdmin = (PulsarAdminWithFunctions) admin; localRunner = new LocalRunner(); creater = new CreateFunction(); deleter = new DeleteFunction(); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index 63caa0d2d3a7e..c25c42e4b3dfc 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -29,6 +29,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.admin.PulsarAdminWithFunctions; import org.apache.pulsar.client.api.AuthenticationFactory; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; @@ -81,6 +82,7 @@ public class PulsarAdminTool { commandMap.put("persistent", CmdPersistentTopics.class); commandMap.put("non-persistent", CmdNonPersistentTopics.class); commandMap.put("resource-quotas", CmdResourceQuotas.class); + commandMap.put("functions", CmdFunctions.class); } private void setupCommands(BiFunction adminFactory) { @@ -103,7 +105,7 @@ private void setupCommands(BiFunction { try { - return new PulsarAdmin(url, config); + return new PulsarAdminWithFunctions(url, config); } catch (Exception ex) { System.err.println(ex.getClass() + ": " + ex.getMessage()); System.exit(1); @@ -173,7 +175,36 @@ public static void main(String[] args) throws Exception { PulsarAdminTool tool = new PulsarAdminTool(properties); - if (tool.run(Arrays.copyOfRange(args, 1, args.length))) { + int cmdPos; + for (cmdPos = 1; cmdPos < args.length; cmdPos++) { + if (tool.commandMap.containsKey(args[cmdPos])) { + break; + } + } + + ++cmdPos; + boolean isLocalRun = false; + if (cmdPos < args.length) { + isLocalRun = "localrun" == args[cmdPos].toLowerCase(); + } + + BiFunction adminFactory; + if (isLocalRun) { + // bypass constructing admin client + adminFactory = (url, config) -> null; + } else { + adminFactory = (url, config) -> { + try { + return new PulsarAdminWithFunctions(url, config); + } catch (Exception ex) { + System.err.println(ex.getClass() + ": " + ex.getMessage()); + System.exit(1); + return null; + } + }; + } + + if (tool.run(Arrays.copyOfRange(args, 1, args.length), adminFactory)) { System.exit(0); } else { System.exit(1); diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/Functions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/Functions.java similarity index 100% rename from pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/Functions.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/Functions.java diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/PulsarFunctionsAdmin.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/PulsarAdminWithFunctions.java similarity index 88% rename from pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/PulsarFunctionsAdmin.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/PulsarAdminWithFunctions.java index 43ede46808e30..86b10be14f018 100644 --- a/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/PulsarFunctionsAdmin.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/PulsarAdminWithFunctions.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -29,10 +29,10 @@ /** - * Pulsar client admin API client. + * Pulsar client admin client with functions support. */ -public class PulsarFunctionsAdmin extends PulsarAdmin { - private static final Logger LOG = LoggerFactory.getLogger(PulsarFunctionsAdmin.class); +public class PulsarAdminWithFunctions extends PulsarAdmin { + private static final Logger LOG = LoggerFactory.getLogger(PulsarAdminWithFunctions.class); private final Functions functions; private final ClientConfigurationData clientConf; @@ -47,7 +47,7 @@ public class PulsarFunctionsAdmin extends PulsarAdmin { * @param pulsarConfig * the ClientConfiguration object to be used to talk with Pulsar */ - public PulsarFunctionsAdmin(URL serviceUrl, ClientConfigurationData pulsarConfig) throws PulsarClientException { + public PulsarAdminWithFunctions(URL serviceUrl, ClientConfigurationData pulsarConfig) throws PulsarClientException { super(serviceUrl, pulsarConfig); this.functions = new FunctionsImpl(web, auth); this.clientConf = pulsarConfig; diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java similarity index 91% rename from pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java rename to pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java index 4ded3e023e148..ae216945f4777 100644 --- a/pulsar-functions/cli/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/internal/FunctionsImpl.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.client.admin.internal; -import com.google.protobuf.util.JsonFormat; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.admin.Functions; import org.apache.pulsar.client.admin.PulsarAdminException; @@ -26,6 +25,7 @@ import org.apache.pulsar.common.policies.data.*; import org.apache.pulsar.functions.proto.Function.FunctionConfig; import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatusList; +import org.apache.pulsar.functions.utils.Utils; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; @@ -72,7 +72,7 @@ public FunctionConfig getFunction(String tenant, String namespace, String functi } String jsonResponse = response.readEntity(String.class); FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - JsonFormat.parser().merge(jsonResponse, functionConfigBuilder); + Utils.mergeJson(jsonResponse, functionConfigBuilder); return functionConfigBuilder.build(); } catch (Exception e) { throw getApiException(e); @@ -89,7 +89,7 @@ public FunctionStatusList getFunctionStatus( } String jsonResponse = response.readEntity(String.class); FunctionStatusList.Builder functionStatusBuilder = FunctionStatusList.newBuilder(); - JsonFormat.parser().merge(jsonResponse, functionStatusBuilder); + Utils.mergeJson(jsonResponse, functionStatusBuilder); return functionStatusBuilder.build(); } catch (Exception e) { throw getApiException(e); @@ -103,8 +103,9 @@ public void createFunction(FunctionConfig functionConfig, String fileName) throw mp.bodyPart(new FileDataBodyPart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM_TYPE)); - mp.bodyPart(new FormDataBodyPart("functionConfig", JsonFormat.printer().print(functionConfig), - MediaType.APPLICATION_JSON_TYPE)); + mp.bodyPart(new FormDataBodyPart("functionConfig", + Utils.printJson(functionConfig), + MediaType.APPLICATION_JSON_TYPE)); request(functions.path(functionConfig.getTenant()).path(functionConfig.getNamespace()).path(functionConfig.getName())) .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA), ErrorData.class); } catch (Exception e) { @@ -129,8 +130,9 @@ public void updateFunction(FunctionConfig functionConfig, String fileName) throw if (fileName != null) { mp.bodyPart(new FileDataBodyPart("data", new File(fileName), MediaType.APPLICATION_OCTET_STREAM_TYPE)); } - mp.bodyPart(new FormDataBodyPart("functionConfig", JsonFormat.printer().print(functionConfig), - MediaType.APPLICATION_JSON_TYPE)); + mp.bodyPart(new FormDataBodyPart("functionConfig", + Utils.printJson(functionConfig), + MediaType.APPLICATION_JSON_TYPE)); request(functions.path(functionConfig.getTenant()).path(functionConfig.getNamespace()).path(functionConfig.getName())) .put(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA), ErrorData.class); } catch (Exception e) { diff --git a/pulsar-functions/cli/pom.xml b/pulsar-functions/cli/pom.xml deleted file mode 100644 index 2345e250782fc..0000000000000 --- a/pulsar-functions/cli/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 2.0.0-incubating-SNAPSHOT - - - pulsar-functions-cli - Pulsar Functions :: CLI - - - - - ${project.groupId} - pulsar-functions-runtime - ${project.version} - - - - org.glassfish.jersey.media - jersey-media-json-jackson - - - - org.glassfish.jersey.core - jersey-client - - - - org.glassfish.jersey.media - jersey-media-multipart - - - - - diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/FunctionsTool.java b/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/FunctionsTool.java deleted file mode 100644 index f3eeaee46643a..0000000000000 --- a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/FunctionsTool.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.admin.cli; - -import java.net.URL; -import java.util.function.BiFunction; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.admin.PulsarAdmin; -import org.apache.pulsar.client.admin.PulsarFunctionsAdmin; - -import java.io.FileInputStream; -import java.util.Arrays; -import java.util.Properties; -import org.apache.pulsar.client.api.ClientConfiguration; -import org.apache.pulsar.client.impl.conf.ClientConfigurationData; - -/** - * TODO: merge this into {@link PulsarAdminTool}. - */ -public class FunctionsTool extends PulsarAdminTool { - - FunctionsTool(Properties properties) throws Exception { - super(properties); - commandMap.put("functions", CmdFunctions.class); - - // in pulsar functions tool, webServiceUrl points to functions rest endpoint, - // which can not be used for constructing pulsar client. we have to use - // `serviceUrl` defined in the properties file. - config.setServiceUrl(properties.getProperty("serviceUrl")); - } - - public static void main(String[] args) throws Exception { - String configFile = args[0]; - Properties properties = new Properties(); - - if (configFile != null) { - FileInputStream fis = null; - try { - fis = new FileInputStream(configFile); - properties.load(fis); - } finally { - if (fis != null) - fis.close(); - } - } - - FunctionsTool tool = new FunctionsTool(properties); - - int cmdPos; - for (cmdPos = 1; cmdPos < args.length; cmdPos++) { - if (tool.commandMap.containsKey(args[cmdPos])) { - break; - } - } - ++cmdPos; - boolean isLocalRun = false; - if (cmdPos < args.length) { - isLocalRun = "localrun" == args[cmdPos].toLowerCase(); - } - - BiFunction adminFactory; - if (isLocalRun) { - // bypass constructing admin client - adminFactory = (url, config) -> null; - } else { - adminFactory = (url, config) -> { - try { - return new PulsarFunctionsAdmin(url, config); - } catch (Exception ex) { - System.err.println(ex.getClass() + ": " + ex.getMessage()); - System.exit(1); - return null; - } - }; - } - - if (tool.run(Arrays.copyOfRange(args, 1, args.length), adminFactory)) { - System.exit(0); - } else { - System.exit(1); - } - } -} diff --git a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/package-info.java b/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/package-info.java deleted file mode 100644 index 56c353e45747e..0000000000000 --- a/pulsar-functions/cli/src/main/java/org/apache/pulsar/admin/cli/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -/** - * Pulsar Functions CLI. - * - * TODO: move this to `pulsar-client-tools` after merged back to the apache repo. - */ -package org.apache.pulsar.admin.cli; \ No newline at end of file diff --git a/pulsar-functions/cli/src/test/resources/log4j2.yml b/pulsar-functions/cli/src/test/resources/log4j2.yml deleted file mode 100644 index 807c8b4657b77..0000000000000 --- a/pulsar-functions/cli/src/test/resources/log4j2.yml +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -Configuration: - name: test - - Appenders: - - # Console - Console: - name: Console - target: SYSTEM_OUT - PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - - Loggers: - - Root: - level: info - AppenderRef: - - ref: Console diff --git a/pulsar-functions/conf/example_logging.yml b/pulsar-functions/conf/example_logging.yml deleted file mode 100644 index 40936cb8458bc..0000000000000 --- a/pulsar-functions/conf/example_logging.yml +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -tenant: "test" -namespace: "test-namespace" -name: "example" -className: "org.apache.pulsar.functions.api.examples.LoggingFunction" -sourceTopic: "persistent://sample/standalone/ns1/test_src" -sinkTopic: "persistent://sample/standalone/ns1/test_result" -inputSerdeClassName: "org.apache.pulsar.functions.runtime.serde.Utf8StringSerDe" -outputSerdeClassName: "org.apache.pulsar.functions.runtime.serde.Utf8StringSerDe" diff --git a/pulsar-functions/dist/pom.xml b/pulsar-functions/dist/pom.xml index 8e0a55eefafda..35066c23d5edf 100644 --- a/pulsar-functions/dist/pom.xml +++ b/pulsar-functions/dist/pom.xml @@ -40,12 +40,6 @@ ${project.version} - - org.apache.pulsar - pulsar-functions-cli - ${project.version} - - org.apache.pulsar pulsar-functions-api-examples diff --git a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java index e4ead652d20bc..06e0cb2b31ef3 100644 --- a/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java +++ b/pulsar-functions/metrics/src/main/java/org/apache/pulsar/functions/metrics/sink/FileSink.java @@ -26,11 +26,11 @@ import java.util.Map; import java.util.logging.Logger; -import com.google.protobuf.util.JsonFormat; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.metrics.MetricsSink; import org.apache.pulsar.functions.utils.FunctionConfigUtils; +import org.apache.pulsar.functions.utils.Utils; /** * A metrics sink that writes to a file in json format @@ -94,7 +94,7 @@ public void processRecord(InstanceCommunication.MetricsData record, Function.Fun } try { - String metrics = JsonFormat.printer().print(record); + String metrics = Utils.printJson(record); writer.print(metrics); } catch (Exception ex) { } diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 2a6daac208740..5aacec56f179a 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -41,7 +41,6 @@ worker worker-shaded worker-runner - cli java-examples dist diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index b5cb83e4c8bb4..242a3fc51fdc5 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -35,7 +35,7 @@ ${project.groupId} - pulsar-client-tools-shaded + pulsar-client-admin-shaded-for-functions ${project.version} diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java index 562f547811c65..c715b033dc94c 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfigUtils.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.protobuf.util.JsonFormat; import org.apache.pulsar.functions.proto.Function.FunctionConfig; import java.io.File; @@ -41,7 +40,7 @@ public static String convertYamlToJson(File file) throws IOException { public static FunctionConfig.Builder loadConfig(File file) throws IOException { String json = convertYamlToJson(file); FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - JsonFormat.parser().merge(json, functionConfigBuilder); + Utils.mergeJson(json, functionConfigBuilder); return functionConfigBuilder; } diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/Utils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/Utils.java index 52ee7e5fcd205..7a15893b23303 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/Utils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/Utils.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.functions.utils; +import com.google.protobuf.AbstractMessage.Builder; +import com.google.protobuf.MessageOrBuilder; +import com.google.protobuf.util.JsonFormat; +import java.io.IOException; import lombok.AccessLevel; import lombok.NoArgsConstructor; import org.apache.pulsar.client.api.MessageId; @@ -49,4 +53,12 @@ public static final MessageId getMessageId(long sequenceId) { return new MessageIdImpl(ledgerId, entryId, -1); } + public static String printJson(MessageOrBuilder msg) throws IOException { + return JsonFormat.printer().print(msg); + } + + public static void mergeJson(String json, Builder builder) throws IOException { + JsonFormat.parser().merge(json, builder); + } + } diff --git a/pulsar-functions/worker-shaded/pom.xml b/pulsar-functions/worker-shaded/pom.xml index 670c8b709f91e..72434312519af 100644 --- a/pulsar-functions/worker-shaded/pom.xml +++ b/pulsar-functions/worker-shaded/pom.xml @@ -44,10 +44,10 @@ org.apache.bookkeeper bookkeeper-common - + org.apache.pulsar - pulsar-client-tools-shaded + pulsar-client-admin-shaded-for-functions diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index ef8a7e63d9f48..1d65078a1029f 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -19,8 +19,7 @@ package org.apache.pulsar.functions.worker; import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; +import java.io.IOException; import lombok.extern.slf4j.Slf4j; import org.apache.distributedlog.api.namespace.Namespace; import org.apache.pulsar.client.api.MessageId; @@ -293,8 +292,8 @@ public InstanceCommunication.FunctionStatus getFunctionInstanceStatus(String ten InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder(); try { - JsonFormat.parser().merge(jsonResponse, functionStatusBuilder); - } catch (InvalidProtocolBufferException e) { + org.apache.pulsar.functions.utils.Utils.mergeJson(jsonResponse, functionStatusBuilder); + } catch (IOException e) { log.warn("Got invalid function status response from {}", workerInfo, e); throw new RuntimeException(e); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 4eda822ef2bcd..7e49b5d7af81b 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -21,8 +21,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; import java.io.IOException; import java.io.InputStream; import java.util.Collection; @@ -239,7 +237,7 @@ public Response deregisterFunction(final @PathParam("tenant") String tenant, public Response getFunctionInfo(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) - throws InvalidProtocolBufferException { + throws IOException { // validate parameters try { @@ -262,7 +260,7 @@ public Response getFunctionInfo(final @PathParam("tenant") String tenant, } FunctionMetaData functionMetaData = functionMetaDataManager.getFunctionMetaData(tenant, namespace, functionName); - String functionConfigJson = JsonFormat.printer().print(functionMetaData.getFunctionConfig()); + String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionMetaData.getFunctionConfig()); return Response.status(Status.OK).entity(functionConfigJson).build(); } @@ -302,11 +300,11 @@ public Response getFunctionInstanceStatus(final @PathParam("tenant") String tena log.error("Got Exception Getting Status", e); FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder(); functionStatusBuilder.setRunning(false); - String functionConfigJson = JsonFormat.printer().print(functionStatusBuilder.build()); + String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build()); return Response.status(Status.OK).entity(functionConfigJson).build(); } - String jsonResponse = JsonFormat.printer().print(functionStatus); + String jsonResponse = org.apache.pulsar.functions.utils.Utils.printJson(functionStatus); return Response.status(Status.OK).entity(jsonResponse).build(); } @@ -344,11 +342,11 @@ public Response getFunctionStatus(final @PathParam("tenant") String tenant, log.error("Got Exception Getting Status", e); FunctionStatus.Builder functionStatusBuilder = FunctionStatus.newBuilder(); functionStatusBuilder.setRunning(false); - String functionConfigJson = JsonFormat.printer().print(functionStatusBuilder.build()); + String functionConfigJson = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusBuilder.build()); return Response.status(Status.OK).entity(functionConfigJson).build(); } - String jsonResponse = JsonFormat.printer().print(functionStatusList); + String jsonResponse = org.apache.pulsar.functions.utils.Utils.printJson(functionStatusList); return Response.status(Status.OK).entity(jsonResponse).build(); } @@ -518,7 +516,7 @@ private FunctionConfig validateUpdateRequestParams(String tenant, } try { FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); - JsonFormat.parser().merge(functionConfigJson, functionConfigBuilder); + org.apache.pulsar.functions.utils.Utils.mergeJson(functionConfigJson, functionConfigBuilder); FunctionConfig functionConfig = functionConfigBuilder.build(); List missingFields = new LinkedList<>(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java index 88cd1a4449735..25ecf0a9329a1 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2Resource.java @@ -18,7 +18,6 @@ */ package org.apache.pulsar.functions.worker.rest.api.v2; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.functions.worker.rest.FunctionApiResource; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -85,7 +84,7 @@ public Response deregisterFunction(final @PathParam("tenant") String tenant, public Response getFunctionInfo(final @PathParam("tenant") String tenant, final @PathParam("namespace") String namespace, final @PathParam("functionName") String functionName) - throws InvalidProtocolBufferException { + throws IOException { return functions.getFunctionInfo( tenant, namespace, functionName); } diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java index 8bce7ffbfc78f..721d4b4cc7237 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v2/FunctionApiV2ResourceTest.java @@ -31,8 +31,6 @@ import com.google.gson.Gson; import com.google.common.collect.Lists; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; import java.io.IOException; import java.io.InputStream; import java.util.List; @@ -131,7 +129,7 @@ public void setup() { // @Test - public void testRegisterFunctionMissingTenant() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingTenant() throws IOException { testRegisterFunctionMissingArguments( null, namespace, @@ -148,7 +146,7 @@ public void testRegisterFunctionMissingTenant() throws InvalidProtocolBufferExce } @Test - public void testRegisterFunctionMissingNamespace() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingNamespace() throws IOException { testRegisterFunctionMissingArguments( tenant, null, @@ -165,7 +163,7 @@ public void testRegisterFunctionMissingNamespace() throws InvalidProtocolBufferE } @Test - public void testRegisterFunctionMissingFunctionName() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingFunctionName() throws IOException { testRegisterFunctionMissingArguments( tenant, namespace, @@ -182,7 +180,7 @@ public void testRegisterFunctionMissingFunctionName() throws InvalidProtocolBuff } @Test - public void testRegisterFunctionMissingPackage() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingPackage() throws IOException { testRegisterFunctionMissingArguments( tenant, namespace, @@ -199,7 +197,7 @@ public void testRegisterFunctionMissingPackage() throws InvalidProtocolBufferExc } @Test - public void testRegisterFunctionMissingPackageDetails() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingPackageDetails() throws IOException { testRegisterFunctionMissingArguments( tenant, namespace, @@ -216,7 +214,7 @@ public void testRegisterFunctionMissingPackageDetails() throws InvalidProtocolBu } @Test - public void testRegisterFunctionMissingSourceTopic() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingSourceTopic() throws IOException { testRegisterFunctionMissingArguments( tenant, namespace, @@ -233,7 +231,7 @@ public void testRegisterFunctionMissingSourceTopic() throws InvalidProtocolBuffe } @Test - public void testRegisterFunctionMissingInputSerde() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingInputSerde() throws IOException { testRegisterFunctionMissingArguments( tenant, namespace, @@ -250,7 +248,7 @@ public void testRegisterFunctionMissingInputSerde() throws InvalidProtocolBuffer } @Test - public void testRegisterFunctionMissingClassName() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingClassName() throws IOException { testRegisterFunctionMissingArguments( tenant, namespace, @@ -267,7 +265,7 @@ public void testRegisterFunctionMissingClassName() throws InvalidProtocolBufferE } @Test - public void testRegisterFunctionMissingParallelism() throws InvalidProtocolBufferException { + public void testRegisterFunctionMissingParallelism() throws IOException { testRegisterFunctionMissingArguments( tenant, namespace, @@ -296,7 +294,7 @@ private void testRegisterFunctionMissingArguments( String className, Integer parallelism, String missingFieldName - ) throws InvalidProtocolBufferException { + ) throws IOException { FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); if (tenant != null) { functionConfigBuilder.setTenant(tenant); @@ -330,7 +328,7 @@ private void testRegisterFunctionMissingArguments( function, inputStream, details, - JsonFormat.printer().print(functionConfig)); + org.apache.pulsar.functions.utils.Utils.printJson(functionConfig)); assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus()); if (missingFieldName.equals("parallelism")) { @@ -340,7 +338,7 @@ private void testRegisterFunctionMissingArguments( } } - private Response registerDefaultFunction() throws InvalidProtocolBufferException { + private Response registerDefaultFunction() throws IOException { FunctionConfig functionConfig = FunctionConfig.newBuilder() .setTenant(tenant).setNamespace(namespace).setName(function) .setOutput(sinkTopic).putCustomSerdeInputs(sourceTopic, inputSerdeClassName) @@ -352,11 +350,12 @@ private Response registerDefaultFunction() throws InvalidProtocolBufferException namespace, function, mockedInputStream, - mockedFormData, JsonFormat.printer().print(functionConfig)); + mockedFormData, + org.apache.pulsar.functions.utils.Utils.printJson(functionConfig)); } @Test - public void testRegisterExistedFunction() throws InvalidProtocolBufferException { + public void testRegisterExistedFunction() throws IOException { Configurator.setRootLevel(Level.DEBUG); when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(true); @@ -450,7 +449,7 @@ public void testRegisterFunctionInterrupted() throws Exception { // @Test - public void testUpdateFunctionMissingTenant() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingTenant() throws IOException { testUpdateFunctionMissingArguments( null, namespace, @@ -467,7 +466,7 @@ public void testUpdateFunctionMissingTenant() throws InvalidProtocolBufferExcept } @Test - public void testUpdateFunctionMissingNamespace() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingNamespace() throws IOException { testUpdateFunctionMissingArguments( tenant, null, @@ -484,7 +483,7 @@ public void testUpdateFunctionMissingNamespace() throws InvalidProtocolBufferExc } @Test - public void testUpdateFunctionMissingFunctionName() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingFunctionName() throws IOException { testUpdateFunctionMissingArguments( tenant, namespace, @@ -501,7 +500,7 @@ public void testUpdateFunctionMissingFunctionName() throws InvalidProtocolBuffer } @Test - public void testUpdateFunctionMissingPackage() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingPackage() throws IOException { testUpdateFunctionMissingArguments( tenant, namespace, @@ -518,7 +517,7 @@ public void testUpdateFunctionMissingPackage() throws InvalidProtocolBufferExcep } @Test - public void testUpdateFunctionMissingPackageDetails() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingPackageDetails() throws IOException { testUpdateFunctionMissingArguments( tenant, namespace, @@ -535,7 +534,7 @@ public void testUpdateFunctionMissingPackageDetails() throws InvalidProtocolBuff } @Test - public void testUpdateFunctionMissingSourceTopic() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingSourceTopic() throws IOException { testUpdateFunctionMissingArguments( tenant, namespace, @@ -552,7 +551,7 @@ public void testUpdateFunctionMissingSourceTopic() throws InvalidProtocolBufferE } @Test - public void testUpdateFunctionMissingInputSerde() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingInputSerde() throws IOException { testUpdateFunctionMissingArguments( tenant, namespace, @@ -569,7 +568,7 @@ public void testUpdateFunctionMissingInputSerde() throws InvalidProtocolBufferEx } @Test - public void testUpdateFunctionMissingClassName() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingClassName() throws IOException { testUpdateFunctionMissingArguments( tenant, namespace, @@ -585,7 +584,7 @@ public void testUpdateFunctionMissingClassName() throws InvalidProtocolBufferExc "ClassName"); } @Test - public void testUpdateFunctionMissingParallelism() throws InvalidProtocolBufferException { + public void testUpdateFunctionMissingParallelism() throws IOException { testUpdateFunctionMissingArguments( tenant, namespace, @@ -615,7 +614,7 @@ private void testUpdateFunctionMissingArguments( String className, Integer parallelism, String missingFieldName - ) throws InvalidProtocolBufferException { + ) throws IOException { FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder(); if (tenant != null) { functionConfigBuilder.setTenant(tenant); @@ -649,7 +648,7 @@ private void testUpdateFunctionMissingArguments( function, inputStream, details, - JsonFormat.printer().print(functionConfig)); + org.apache.pulsar.functions.utils.Utils.printJson(functionConfig)); assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus()); if (missingFieldName.equals("parallelism")) { @@ -659,7 +658,7 @@ private void testUpdateFunctionMissingArguments( } } - private Response updateDefaultFunction() throws InvalidProtocolBufferException { + private Response updateDefaultFunction() throws IOException { FunctionConfig functionConfig = FunctionConfig.newBuilder() .setTenant(tenant).setNamespace(namespace).setName(function) .setOutput(sinkTopic).putCustomSerdeInputs(sourceTopic, inputSerdeClassName) @@ -671,11 +670,12 @@ private Response updateDefaultFunction() throws InvalidProtocolBufferException { namespace, function, mockedInputStream, - mockedFormData, JsonFormat.printer().print(functionConfig)); + mockedFormData, + org.apache.pulsar.functions.utils.Utils.printJson(functionConfig)); } @Test - public void testUpdateNotExistedFunction() throws InvalidProtocolBufferException { + public void testUpdateNotExistedFunction() throws IOException { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); Response response = updateDefaultFunction(); @@ -903,7 +903,7 @@ private void testGetFunctionMissingArguments( String namespace, String function, String missingFieldName - ) throws InvalidProtocolBufferException { + ) throws IOException { Response response = resource.getFunctionInfo( tenant, namespace, @@ -913,7 +913,7 @@ private void testGetFunctionMissingArguments( assertEquals(new ErrorData(missingFieldName + " is not provided").reason, ((ErrorData) response.getEntity()).reason); } - private Response getDefaultFunctionInfo() throws InvalidProtocolBufferException { + private Response getDefaultFunctionInfo() throws IOException { return resource.getFunctionInfo( tenant, namespace, @@ -921,7 +921,7 @@ private Response getDefaultFunctionInfo() throws InvalidProtocolBufferException } @Test - public void testGetNotExistedFunction() throws InvalidProtocolBufferException { + public void testGetNotExistedFunction() throws IOException { when(mockedManager.containsFunction(eq(tenant), eq(namespace), eq(function))).thenReturn(false); Response response = getDefaultFunctionInfo(); @@ -953,7 +953,9 @@ public void testGetFunctionSuccess() throws Exception { Response response = getDefaultFunctionInfo(); assertEquals(Status.OK.getStatusCode(), response.getStatus()); - assertEquals(JsonFormat.printer().print(functionConfig), response.getEntity()); + assertEquals( + org.apache.pulsar.functions.utils.Utils.printJson(functionConfig), + response.getEntity()); } // From 2648100481431049f5eb3741a1ee4effac83dbe2 Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Mon, 5 Mar 2018 01:26:01 -0800 Subject: [PATCH 12/14] Merge pulsar-functions dist package into pulsar binary distribution --- all/pom.xml | 9 + all/src/assemble/bin.xml | 14 ++ pulsar-functions/bin/pulsar | 192 --------------------- pulsar-functions/bin/pulsar-functions | 148 ---------------- pulsar-functions/conf/client.conf | 27 --- pulsar-functions/conf/function_worker.yml | 46 ----- pulsar-functions/conf/log4j2.yml | 167 ------------------ pulsar-functions/conf/pulsar_env.sh | 60 ------- pulsar-functions/conf/pulsar_tools_env.sh | 60 ------- pulsar-functions/dist/pom.xml | 103 ----------- pulsar-functions/dist/src/assemble/bin.xml | 83 --------- pulsar-functions/dist/src/assemble/src.xml | 92 ---------- pulsar-functions/pom.xml | 2 - pulsar-functions/worker-runner/pom.xml | 111 ------------ 14 files changed, 23 insertions(+), 1091 deletions(-) delete mode 100755 pulsar-functions/bin/pulsar delete mode 100755 pulsar-functions/bin/pulsar-functions delete mode 100644 pulsar-functions/conf/client.conf delete mode 100644 pulsar-functions/conf/function_worker.yml delete mode 100644 pulsar-functions/conf/log4j2.yml delete mode 100644 pulsar-functions/conf/pulsar_env.sh delete mode 100755 pulsar-functions/conf/pulsar_tools_env.sh delete mode 100644 pulsar-functions/dist/pom.xml delete mode 100644 pulsar-functions/dist/src/assemble/bin.xml delete mode 100644 pulsar-functions/dist/src/assemble/src.xml delete mode 100644 pulsar-functions/worker-runner/pom.xml diff --git a/all/pom.xml b/all/pom.xml index 0c02d51ef2e04..6273f4bf4449a 100644 --- a/all/pom.xml +++ b/all/pom.xml @@ -104,6 +104,15 @@ pulsar-zookeeper ${project.version} + + + + org.apache.pulsar + pulsar-functions-api-examples + ${project.version} + + provided + diff --git a/all/src/assemble/bin.xml b/all/src/assemble/bin.xml index b295bab9ea0bf..3a994acf9d7be 100644 --- a/all/src/assemble/bin.xml +++ b/all/src/assemble/bin.xml @@ -43,6 +43,10 @@ ${basedir}/licenses + + ${basedir}/../pulsar-functions/runtime/target/python-instance + instances/python-instance + @@ -68,6 +72,16 @@ . 644 + + ${basedir}/../pulsar-functions/runtime/target/java-instance.jar + java-instance.jar + instances + + + ${basedir}/../pulsar-functions/java-examples/target/pulsar-functions-api-examples.jar + api-examples.jar + examples + diff --git a/pulsar-functions/bin/pulsar b/pulsar-functions/bin/pulsar deleted file mode 100755 index ba219a1b617e3..0000000000000 --- a/pulsar-functions/bin/pulsar +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BINDIR=$(dirname "$0") -PULSAR_HOME=`cd $BINDIR/..;pwd` - -DEFAULT_WORKER_CONF=$PULSAR_HOME/conf/function_worker.yml -DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yml -DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar -JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"} -DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py -PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"} - -if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] -then - . "$PULSAR_HOME/conf/pulsar_env.sh" -fi - -# Check for the java to use -if [[ -z $JAVA_HOME ]]; then - JAVA=$(which java) - if [ $? != 0 ]; then - echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 - exit 1 - fi -else - JAVA=$JAVA_HOME/bin/java -fi - -# exclude tests jar -RELEASE_JAR=`ls $PULSAR_HOME/lib/*pulsar-functions-worker*.jar 2> /dev/null | grep -v tests | tail -1` -if [ $? == 0 ]; then - PULSAR_JAR=$RELEASE_JAR -fi - -# exclude tests jar -if [ -z "$PULSAR_JAR" ]; then - BUILT_JAR=`ls $PULSAR_HOME/worker-runner/target/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` - if [ -z "${BUILT_JAR}" ]; then - echo "\nCouldn't find pulsar jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - PULSAR_JAR=$BUILT_JAR -fi - -# find the java instance location -if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then - # didn't find a released jar, then search the built jar - BUILT_JAVA_INSTANCE_JAR="${PULSAR_HOME}/runtime/target/java-instance.jar" - if [ -z "${BUILT_JAVA_INSTANCE_JAR}" ]; then - echo "\nCouldn't find pulsar java instance jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR} -fi - -# find the python instance location -if [ ! -f "${PY_INSTANCE_FILE}" ]; then - # didn't find a released python instance, then search the built python instance - BUILT_PY_INSTANCE_FILE="${PULSAR_HOME}/runtime/target/python-instance/python_instance_main.py" - if [ -z "${BUILT_PY_INSTANCE_FILE}" ]; then - echo "\nCouldn't find pulsar python instance."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE} -fi - -pulsar_help() { - cat < -where command is one of: - worker Run a function worker server - - help This help message - -or command is the full name of a class with a defined main() method. - -Environment variables: - PULSAR_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) - PULSAR_WORKER_CONF Configuration file for function worker (default: $DEFAULT_WORKER_CONF) - PULSAR_EXTRA_OPTS Extra options to be passed to the jvm - PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath - PULSAR_PID_DIR Folder where the pulsar server PID file should be stored - PULSAR_STOP_TIMEOUT Wait time before forcefully kill the pulsar server instance, if the stop is not successful - PULSAR_JAVA_INSTANCE_JAR The java instance jar that pulsar worker uses to run functions (default: $DEFAULT_JAVA_INSTANCE_JAR) - PULSAR_PY_INSTANCE_FILE The python instance that pulsar worker uses to run functions (default: $DEFAULT_PY_INSTANCE_FILE) - -These variable can also be set in conf/pulsar_env.sh -EOF -} - -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${PULSAR_HOME}/worker-runner/target/classpath_shaded.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${PULSAR_HOME}/worker-runner/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null - fi - PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$PULSAR_HOME/lib" ]; then - PULSAR_CLASSPATH=$PULSAR_CLASSPATH:$PULSAR_HOME/lib/* -else - add_maven_deps_to_classpath -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - pulsar_help; - exit 1; -fi - -# get arguments -COMMAND=$1 -shift - -if [ -z "$PULSAR_WORKER_CONF" ]; then - PULSAR_WORKER_CONF=$DEFAULT_WORKER_CONF -fi - -if [ -z "$PULSAR_LOG_CONF" ]; then - PULSAR_LOG_CONF=$DEFAULT_LOG_CONF -fi - -PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH" -PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH" -OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`" - -# Ensure we can read bigger content from ZK. (It might be -# rarely needed when trying to list many z-nodes under a -# directory) -OPTS="$OPTS -Djute.maxbuffer=10485760 -Djava.net.preferIPv4Stack=true" - -OPTS="-cp $PULSAR_CLASSPATH $OPTS" - -OPTS="$OPTS $PULSAR_EXTRA_OPTS" - -# log directory & file -PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} -PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} -PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} -PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} - -#Configure log configuration system properties -OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" -OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" -OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" -OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT" -OPTS="$OPTS -Dpulsar.functions.process.container.log.dir=$PULSAR_LOG_DIR" - -# instance -OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}" -OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}" - -#Change to PULSAR_HOME to support relative paths -cd "$PULSAR_HOME" -if [ $COMMAND == "worker" ]; then - PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-function-worker.log"} - exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF $@ -elif [ $COMMAND == "help" ]; then - pulsar_help; -else - exec $JAVA $OPTS $COMMAND $@ -fi diff --git a/pulsar-functions/bin/pulsar-functions b/pulsar-functions/bin/pulsar-functions deleted file mode 100755 index ecd12379efcba..0000000000000 --- a/pulsar-functions/bin/pulsar-functions +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BINDIR=$(dirname "$0") -PULSAR_HOME=`cd $BINDIR/..;pwd` - -DEFAULT_CLIENT_CONF=$PULSAR_HOME/conf/client.conf -DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yml -DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar -JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"} -DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py -PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"} - -if [ -f "$PULSAR_HOME/conf/pulsar_tools_env.sh" ] -then - . "$PULSAR_HOME/conf/pulsar_tools_env.sh" -fi - -# Check for the java to use -if [[ -z $JAVA_HOME ]]; then - JAVA=$(which java) - if [ $? != 0 ]; then - echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 - exit 1 - fi -else - JAVA=$JAVA_HOME/bin/java -fi - -# exclude tests jar -RELEASE_JAR=`ls $PULSAR_HOME/lib/*pulsar-functions-cli*.jar 2> /dev/null | grep -v tests | tail -1` -if [ $? == 0 ]; then - PULSAR_JAR=$RELEASE_JAR -fi - -# exclude tests jar -if [ -z "$PULSAR_JAR" ]; then - BUILT_JAR=`ls $PULSAR_HOME/cli/target/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` - if [ -z "${BUILT_JAR}" ]; then - echo "\nCouldn't find pulsar jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - PULSAR_JAR=$BUILT_JAR -fi - -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${PULSAR_HOME}/cli/target/classpath_shaded.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${PULSAR_HOME}/cli/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null - fi - PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$PULSAR_HOME/lib" ]; then - PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*" -else - add_maven_deps_to_classpath -fi - -if [ -z "$PULSAR_CLIENT_CONF" ]; then - PULSAR_CLIENT_CONF=$DEFAULT_CLIENT_CONF -fi -if [ -z "$PULSAR_LOG_CONF" ]; then - PULSAR_LOG_CONF=$DEFAULT_LOG_CONF -fi - -PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH" -PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH" -OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`" -OPTS="$OPTS -Djava.net.preferIPv4Stack=true" - -OPTS="-cp $PULSAR_CLASSPATH $OPTS" - -OPTS="$OPTS $PULSAR_EXTRA_OPTS" - -# log directory & file -PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} -PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} -PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} -PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} - -#Configure log configuration system properties -OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" -OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" -OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" -OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT" - -# find the java instance location -if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then - # didn't find a released jar, then search the built jar - BUILT_JAVA_INSTANCE_JAR="${PULSAR_HOME}/runtime/target/java-instance.jar" - if [ -f "${BUILT_JAVA_INSTANCE_JAR}" ]; then - JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR} - else - echo "\nCouldn't find pulsar java instance jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi -fi - -# find the python instance location -if [ ! -f "${PY_INSTANCE_FILE}" ]; then - # didn't find a released python instance, then search the built python instance - BUILT_PY_INSTANCE_FILE="${PULSAR_HOME}/runtime/target/python-instance/python_instance_main.py" - if [ -f "${BUILT_PY_INSTANCE_FILE}" ]; then - PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE} - else - echo "\nCouldn't find pulsar python instance."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi -fi - -# functions -OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}" -OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}" - -#Change to PULSAR_HOME to support relative paths -cd "$PULSAR_HOME" - -exec $JAVA $OPTS org.apache.pulsar.admin.cli.FunctionsTool $PULSAR_CLIENT_CONF "$@" diff --git a/pulsar-functions/conf/client.conf b/pulsar-functions/conf/client.conf deleted file mode 100644 index fabe68c3e011c..0000000000000 --- a/pulsar-functions/conf/client.conf +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Pulsar Client configuration -webServiceUrl=http://localhost:6750/ -brokerServiceUrl=pulsar://localhost:6650/ -#authPlugin= -#authParams= -#useTls= -#tlsAllowInsecureConnection -#tlsTrustCertsFilePath diff --git a/pulsar-functions/conf/function_worker.yml b/pulsar-functions/conf/function_worker.yml deleted file mode 100644 index 2bc51d1757c7c..0000000000000 --- a/pulsar-functions/conf/function_worker.yml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -workerId: standalone -workerHostname: localhost -workerPort: 6750 -functionMetadataTopicName: metadata -functionMetadataSnapshotsTopicPath: snapshots -clusterCoordinationTopicName: coordinate -pulsarFunctionsNamespace: sample/standalone/functions -pulsarServiceUrl: pulsar://localhost:6650 -pulsarWebServiceUrl: http://localhost:8080 -numFunctionPackageReplicas: 1 -downloadDirectory: /tmp/pulsar_functions -metricsConfig: - metricsSinkClassName: org.apache.pulsar.functions.metrics.sink.PrometheusSink - metricsCollectionInterval: 30 - metricsSinkConfig: - path: /metrics - port: 9099 -# threadContainerFactory: -# threadGroupName: "Thread Function Container Group" -processContainerFactory: - logDirectory: - -schedulerClassName: "org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler" -functionAssignmentTopicName: "assignments" -failureCheckFreqMs: 30000 -rescheduleTimeoutMs: 60000 -initialBrokerReconnectMaxRetries: 60 diff --git a/pulsar-functions/conf/log4j2.yml b/pulsar-functions/conf/log4j2.yml deleted file mode 100644 index 47b8dcb2372d2..0000000000000 --- a/pulsar-functions/conf/log4j2.yml +++ /dev/null @@ -1,167 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -Configuration: - name: pulsar-functions - monitorInterval: 30 - - Properties: - Property: - - name: "pulsar.log.dir" - value: "logs" - - name: "pulsar.log.file" - value: "pulsar-functions.log" - - name: "pulsar.log.appender" - value: "RoutingAppender" - - name: "pulsar.log.level" - value: "info" - - name: "pulsar.routing.appender.default" - value: "Console" - - name: "bk.log.level" - value: "info" - - Appenders: - - # Console - Console: - name: Console - target: SYSTEM_OUT - PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - - # Rolling file appender configuration - RollingFile: - name: RollingFile - fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}" - filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz" - immediateFlush: true - PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - Policies: - TimeBasedTriggeringPolicy: - interval: 1 - modulate: true - SizeBasedTriggeringPolicy: - size: 1 GB - # Trigger every day at midnight that also scan - # roll-over strategy that deletes older file - CronTriggeringPolicy: - schedule: "0 0 0 * * ?" - # Delete file older than 30days - DefaultRolloverStrategy: - Delete: - basePath: ${sys:pulsar.log.dir} - maxDepth: 2 - IfFileName: - glob: "*/${sys:pulsar.log.file}*log.gz" - IfLastModified: - age: 30d - - # Rolling file appender configuration for bk - RollingRandomAccessFile: - name: BkRollingFile - fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk" - filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz" - immediateFlush: true - PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - Policies: - TimeBasedTriggeringPolicy: - interval: 1 - modulate: true - SizeBasedTriggeringPolicy: - size: 1 GB - # Trigger every day at midnight that also scan - # roll-over strategy that deletes older file - CronTriggeringPolicy: - schedule: "0 0 0 * * ?" - # Delete file older than 30days - DefaultRolloverStrategy: - Delete: - basePath: ${sys:pulsar.log.dir} - maxDepth: 2 - IfFileName: - glob: "*/${sys:pulsar.log.file}.bk*log.gz" - IfLastModified: - age: 30d - - # Routing - Routing: - name: RoutingAppender - Routes: - pattern: "$${ctx:function}" - Route: - - - Routing: - name: InstanceRoutingAppender - Routes: - pattern: "$${ctx:instance}" - Route: - - - RollingFile: - name: "Rolling-${ctx:function}" - fileName : "${sys:pulsar.log.dir}/functions/${ctx:function}/function.log" - filePattern : "${sys:pulsar.log.dir}/functions/${ctx:function}-%d{MM-dd-yyyy}-%i.log.gz" - PatternLayout: - Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance: %X{instance}] %logger{1} - %msg%n" - Policies: - TimeBasedTriggeringPolicy: - interval: 1 - modulate: true - SizeBasedTriggeringPolicy: - size: "20MB" - # Trigger every day at midnight that also scan - # roll-over strategy that deletes older file - CronTriggeringPolicy: - schedule: "0 0 0 * * ?" - # Delete file older than 30days - DefaultRolloverStrategy: - Delete: - basePath: ${sys:pulsar.log.dir} - maxDepth: 2 - IfFileName: - glob: "*/${sys:pulsar.log.file}*log.gz" - IfLastModified: - age: 30d - - ref: "${sys:pulsar.routing.appender.default}" - key: "${ctx:function}" - - ref: "${sys:pulsar.routing.appender.default}" - key: "${ctx:function}" - - Loggers: - - Logger: - name: org.apache.bookkeeper - level: "${sys:bk.log.level}" - additivity: false - AppenderRef: - - ref: BkRollingFile - - Logger: - name: org.apache.distributedlog - level: "${sys:bk.log.level}" - additivity: false - AppenderRef: - - ref: BkRollingFile - - Root: - level: info - AppenderRef: - - ref: "${sys:pulsar.log.appender}" - level: "${sys:pulsar.log.level}" diff --git a/pulsar-functions/conf/pulsar_env.sh b/pulsar-functions/conf/pulsar_env.sh deleted file mode 100644 index f600e6d20a389..0000000000000 --- a/pulsar-functions/conf/pulsar_env.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Set JAVA_HOME here to override the environment setting -# JAVA_HOME= - -# default settings for starting pulsar broker - -# Log4j configuration file -# PULSAR_LOG_CONF= - -# Logs location -# PULSAR_LOG_DIR= - -# Configuration file of settings used in broker server -# PULSAR_BROKER_CONF= - -# Configuration file of settings used in bookie server -# PULSAR_BOOKKEEPER_CONF= - -# Configuration file of settings used in zookeeper server -# PULSAR_ZK_CONF= - -# Configuration file of settings used in global zookeeper server -# PULSAR_GLOBAL_ZK_CONF= - -# Extra options to be passed to the jvm -PULSAR_MEM=" -Xms2g -Xmx2g -XX:MaxDirectMemorySize=4g" - -# Garbage collection options -PULSAR_GC=" -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB" - -# Extra options to be passed to the jvm -PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" - -# Add extra paths to the bookkeeper classpath -# PULSAR_EXTRA_CLASSPATH= - -#Folder where the Bookie server PID file should be stored -#PULSAR_PID_DIR= - -#Wait time before forcefully kill the pulser server instance, if the stop is not successful -#PULSAR_STOP_TIMEOUT= diff --git a/pulsar-functions/conf/pulsar_tools_env.sh b/pulsar-functions/conf/pulsar_tools_env.sh deleted file mode 100755 index 7eca6071d2480..0000000000000 --- a/pulsar-functions/conf/pulsar_tools_env.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Set JAVA_HOME here to override the environment setting -# JAVA_HOME= - -# default settings for starting pulsar broker - -# Log4j configuration file -# PULSAR_LOG_CONF= - -# Logs location -# PULSAR_LOG_DIR= - -# Configuration file of settings used in broker server -# PULSAR_BROKER_CONF= - -# Configuration file of settings used in bookie server -# PULSAR_BOOKKEEPER_CONF= - -# Configuration file of settings used in zookeeper server -# PULSAR_ZK_CONF= - -# Configuration file of settings used in global zookeeper server -# PULSAR_GLOBAL_ZK_CONF= - -# Extra options to be passed to the jvm -PULSAR_MEM=" -Xmx256m -XX:MaxDirectMemorySize=256m" - -# Garbage collection options -PULSAR_GC=" -client " - -# Extra options to be passed to the jvm -PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.leakDetectionLevel=disabled" - -# Add extra paths to the bookkeeper classpath -# PULSAR_EXTRA_CLASSPATH= - -#Folder where the Bookie server PID file should be stored -#PULSAR_PID_DIR= - -#Wait time before forcefully kill the pulser server instance, if the stop is not successful -#PULSAR_STOP_TIMEOUT= diff --git a/pulsar-functions/dist/pom.xml b/pulsar-functions/dist/pom.xml deleted file mode 100644 index 35066c23d5edf..0000000000000 --- a/pulsar-functions/dist/pom.xml +++ /dev/null @@ -1,103 +0,0 @@ - - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 2.0.0-incubating-SNAPSHOT - .. - - - pulsar-functions-dist - Pulsar Functions :: Distribution - pom - - - - org.apache.pulsar - pulsar-functions-worker - ${project.version} - - - - org.apache.pulsar - pulsar-functions-api-examples - ${project.version} - - provided - - - - io.netty - netty-all - - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - - - - - - maven-assembly-plugin - - - functions-dist-assembly - package - - single - - - true - posix - apache-pulsar-functions-${project.version} - - src/assemble/bin.xml - src/assemble/src.xml - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - - diff --git a/pulsar-functions/dist/src/assemble/bin.xml b/pulsar-functions/dist/src/assemble/bin.xml deleted file mode 100644 index b4def906e2e2b..0000000000000 --- a/pulsar-functions/dist/src/assemble/bin.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - bin - - tar.gz - - true - - - true - - - - - ${basedir}/../conf - - - ${basedir}/../bin - 755 - - - ${basedir}/../runtime/target/python-instance - instances/python-instance - - - - - ${basedir}/../runtime/target/java-instance.jar - java-instance.jar - instances - - - ${basedir}/../java-examples/target/pulsar-functions-api-examples.jar - api-examples.jar - examples - - - - - lib - false - compile - false - - ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} - - - - io.netty:netty-common - io.netty:netty-resolver - io.netty:netty-buffer - io.netty:netty-codec-http - io.netty:netty-codec - io.netty:netty-transport - io.netty:netty-handler - io.netty:netty-codec-http - - - - diff --git a/pulsar-functions/dist/src/assemble/src.xml b/pulsar-functions/dist/src/assemble/src.xml deleted file mode 100644 index 0c0526358e515..0000000000000 --- a/pulsar-functions/dist/src/assemble/src.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - src - - tar.gz - - true - - - .. - true - - **/README.md - **/DISCLAIMER - **/LICENSE - **/NOTICE - **/pom.xml - **/src/** - **/pulsar-client-cpp/** - **/conf/** - **/bin/** - **/*.txt - docker/** - dashboard/** - deployment/** - - - .git/** - **/.gitignore - **/.svn - **/*.iws - **/*.ipr - **/*.iml - **/*.cbp - **/*.pyc - **/.classpath - **/.project - **/.settings - **/target/** - **/CMakeFiles/** - **/CMakeCache.txt - **/cmake_install.cmake - pulsar-client-cpp/**/Makefile - pulsar-client-cpp/tests/main - pulsar-client-cpp/examples/SampleAsyncProducer - pulsar-client-cpp/examples/SampleConsumer - pulsar-client-cpp/examples/SampleConsumerListener - pulsar-client-cpp/examples/SampleProducer - pulsar-client-cpp/perf/perfProducer - pulsar-client-cpp/perf/perfConsumer - - **/python/dist/** - **/python/wheelhouse/** - **/python/MANIFEST - **/*.egg-info/** - - **/*.log - **/build/** - **/file:/** - **/SecurityAuth.audit* - **/site/** - **/.idea/** - **/*.a - **/*.so - **/*.so.* - **/*.dylib - - - - diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 5aacec56f179a..829f080ee0536 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -40,9 +40,7 @@ runtime worker worker-shaded - worker-runner java-examples - dist diff --git a/pulsar-functions/worker-runner/pom.xml b/pulsar-functions/worker-runner/pom.xml deleted file mode 100644 index 1e621d5cc6f58..0000000000000 --- a/pulsar-functions/worker-runner/pom.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 2.0.0-incubating-SNAPSHOT - .. - - - pulsar-functions-worker-runner - Pulsar Functions :: Worker Runner - - - - - ${project.groupId} - pulsar-functions-worker-shaded - ${project.parent.version} - - - com.google.protobuf - protobuf-lite - - - com.google.protobuf - protobuf-java - - - com.google.protobuf.nano - protobuf-javanano - - - com.google.protobuf - protobuf-java-util - - - io.grpc - grpc-protobuf - - - io.grpc - grpc-protobuf-lite - - - io.grpc - grpc-protobuf-nano - - - org.apache.pulsar - pulsar-functions-proto - - - org.apache.pulsar - pulsar-functions-utils - - - org.apache.pulsar - pulsar-functions-metrics - - - org.apache.pulsar - pulsar-functions-runtime - - - org.apache.pulsar - pulsar-functions-instance - - - org.apache.pulsar - pulsar-functions-worker - - - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - From ee063956e1350459d7ae6de065cb75a3a2bb721d Mon Sep 17 00:00:00 2001 From: Sijie Guo Date: Mon, 5 Mar 2018 14:30:34 -0800 Subject: [PATCH 13/14] Fix license header issues --- .../test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java | 2 +- .../apache/pulsar/client/admin/PulsarAdminWithFunctions.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java index daf29c52af59c..e57aa84440735 100644 --- a/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java +++ b/pulsar-client-tools-test/src/test/java/org/apache/pulsar/admin/cli/CmdFunctionsTest.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/PulsarAdminWithFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/PulsarAdminWithFunctions.java index 86b10be14f018..29b767ff1aa2a 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/PulsarAdminWithFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/admin/PulsarAdminWithFunctions.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information From aaca4f74e663876cc09f2423ac5455642b40f681 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 5 Mar 2018 16:43:16 -0800 Subject: [PATCH 14/14] Fixed ZK cache test exectutor configuration. Fixes #1338 --- .../java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java index a6333403bebaf..e9bad89f5fdf8 100644 --- a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java +++ b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java @@ -73,7 +73,7 @@ void teardown() throws Exception { @BeforeClass void classSetup() throws Exception { - executor = OrderedScheduler.newSchedulerBuilder().maxTasksInQueue(1).name("ZookeeperCacheTest").build(); + executor = OrderedScheduler.newSchedulerBuilder().numThreads(1).name("ZookeeperCacheTest").build(); scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); }