diff --git a/all/pom.xml b/all/pom.xml index 0c02d51ef2e04..6273f4bf4449a 100644 --- a/all/pom.xml +++ b/all/pom.xml @@ -104,6 +104,15 @@ pulsar-zookeeper ${project.version} + + + + org.apache.pulsar + pulsar-functions-api-examples + ${project.version} + + provided + diff --git a/all/src/assemble/bin.xml b/all/src/assemble/bin.xml index b295bab9ea0bf..3a994acf9d7be 100644 --- a/all/src/assemble/bin.xml +++ b/all/src/assemble/bin.xml @@ -43,6 +43,10 @@ ${basedir}/licenses + + ${basedir}/../pulsar-functions/runtime/target/python-instance + instances/python-instance + @@ -68,6 +72,16 @@ . 644 + + ${basedir}/../pulsar-functions/runtime/target/java-instance.jar + java-instance.jar + instances + + + ${basedir}/../pulsar-functions/java-examples/target/pulsar-functions-api-examples.jar + api-examples.jar + examples + diff --git a/pulsar-functions/bin/pulsar b/pulsar-functions/bin/pulsar deleted file mode 100755 index ba219a1b617e3..0000000000000 --- a/pulsar-functions/bin/pulsar +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BINDIR=$(dirname "$0") -PULSAR_HOME=`cd $BINDIR/..;pwd` - -DEFAULT_WORKER_CONF=$PULSAR_HOME/conf/function_worker.yml -DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yml -DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar -JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"} -DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py -PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"} - -if [ -f "$PULSAR_HOME/conf/pulsar_env.sh" ] -then - . "$PULSAR_HOME/conf/pulsar_env.sh" -fi - -# Check for the java to use -if [[ -z $JAVA_HOME ]]; then - JAVA=$(which java) - if [ $? != 0 ]; then - echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 - exit 1 - fi -else - JAVA=$JAVA_HOME/bin/java -fi - -# exclude tests jar -RELEASE_JAR=`ls $PULSAR_HOME/lib/*pulsar-functions-worker*.jar 2> /dev/null | grep -v tests | tail -1` -if [ $? == 0 ]; then - PULSAR_JAR=$RELEASE_JAR -fi - -# exclude tests jar -if [ -z "$PULSAR_JAR" ]; then - BUILT_JAR=`ls $PULSAR_HOME/worker-runner/target/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` - if [ -z "${BUILT_JAR}" ]; then - echo "\nCouldn't find pulsar jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - PULSAR_JAR=$BUILT_JAR -fi - -# find the java instance location -if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then - # didn't find a released jar, then search the built jar - BUILT_JAVA_INSTANCE_JAR="${PULSAR_HOME}/runtime/target/java-instance.jar" - if [ -z "${BUILT_JAVA_INSTANCE_JAR}" ]; then - echo "\nCouldn't find pulsar java instance jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR} -fi - -# find the python instance location -if [ ! -f "${PY_INSTANCE_FILE}" ]; then - # didn't find a released python instance, then search the built python instance - BUILT_PY_INSTANCE_FILE="${PULSAR_HOME}/runtime/target/python-instance/python_instance_main.py" - if [ -z "${BUILT_PY_INSTANCE_FILE}" ]; then - echo "\nCouldn't find pulsar python instance."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE} -fi - -pulsar_help() { - cat < -where command is one of: - worker Run a function worker server - - help This help message - -or command is the full name of a class with a defined main() method. - -Environment variables: - PULSAR_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF) - PULSAR_WORKER_CONF Configuration file for function worker (default: $DEFAULT_WORKER_CONF) - PULSAR_EXTRA_OPTS Extra options to be passed to the jvm - PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath - PULSAR_PID_DIR Folder where the pulsar server PID file should be stored - PULSAR_STOP_TIMEOUT Wait time before forcefully kill the pulsar server instance, if the stop is not successful - PULSAR_JAVA_INSTANCE_JAR The java instance jar that pulsar worker uses to run functions (default: $DEFAULT_JAVA_INSTANCE_JAR) - PULSAR_PY_INSTANCE_FILE The python instance that pulsar worker uses to run functions (default: $DEFAULT_PY_INSTANCE_FILE) - -These variable can also be set in conf/pulsar_env.sh -EOF -} - -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${PULSAR_HOME}/worker-runner/target/classpath_shaded.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${PULSAR_HOME}/worker-runner/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null - fi - PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$PULSAR_HOME/lib" ]; then - PULSAR_CLASSPATH=$PULSAR_CLASSPATH:$PULSAR_HOME/lib/* -else - add_maven_deps_to_classpath -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - pulsar_help; - exit 1; -fi - -# get arguments -COMMAND=$1 -shift - -if [ -z "$PULSAR_WORKER_CONF" ]; then - PULSAR_WORKER_CONF=$DEFAULT_WORKER_CONF -fi - -if [ -z "$PULSAR_LOG_CONF" ]; then - PULSAR_LOG_CONF=$DEFAULT_LOG_CONF -fi - -PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH" -PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH" -OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`" - -# Ensure we can read bigger content from ZK. (It might be -# rarely needed when trying to list many z-nodes under a -# directory) -OPTS="$OPTS -Djute.maxbuffer=10485760 -Djava.net.preferIPv4Stack=true" - -OPTS="-cp $PULSAR_CLASSPATH $OPTS" - -OPTS="$OPTS $PULSAR_EXTRA_OPTS" - -# log directory & file -PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} -PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} -PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} -PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} - -#Configure log configuration system properties -OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" -OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" -OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" -OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT" -OPTS="$OPTS -Dpulsar.functions.process.container.log.dir=$PULSAR_LOG_DIR" - -# instance -OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}" -OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}" - -#Change to PULSAR_HOME to support relative paths -cd "$PULSAR_HOME" -if [ $COMMAND == "worker" ]; then - PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-function-worker.log"} - exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF $@ -elif [ $COMMAND == "help" ]; then - pulsar_help; -else - exec $JAVA $OPTS $COMMAND $@ -fi diff --git a/pulsar-functions/bin/pulsar-functions b/pulsar-functions/bin/pulsar-functions deleted file mode 100755 index ecd12379efcba..0000000000000 --- a/pulsar-functions/bin/pulsar-functions +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BINDIR=$(dirname "$0") -PULSAR_HOME=`cd $BINDIR/..;pwd` - -DEFAULT_CLIENT_CONF=$PULSAR_HOME/conf/client.conf -DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yml -DEFAULT_JAVA_INSTANCE_JAR=$PULSAR_HOME/instances/java-instance.jar -JAVA_INSTANCE_JAR=${PULSAR_JAVA_INSTANCE_JAR:-"${DEFAULT_JAVA_INSTANCE_JAR}"} -DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_main.py -PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"} - -if [ -f "$PULSAR_HOME/conf/pulsar_tools_env.sh" ] -then - . "$PULSAR_HOME/conf/pulsar_tools_env.sh" -fi - -# Check for the java to use -if [[ -z $JAVA_HOME ]]; then - JAVA=$(which java) - if [ $? != 0 ]; then - echo "Error: JAVA_HOME not set, and no java executable found in $PATH." 1>&2 - exit 1 - fi -else - JAVA=$JAVA_HOME/bin/java -fi - -# exclude tests jar -RELEASE_JAR=`ls $PULSAR_HOME/lib/*pulsar-functions-cli*.jar 2> /dev/null | grep -v tests | tail -1` -if [ $? == 0 ]; then - PULSAR_JAR=$RELEASE_JAR -fi - -# exclude tests jar -if [ -z "$PULSAR_JAR" ]; then - BUILT_JAR=`ls $PULSAR_HOME/cli/target/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1` - if [ -z "${BUILT_JAR}" ]; then - echo "\nCouldn't find pulsar jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi - PULSAR_JAR=$BUILT_JAR -fi - -add_maven_deps_to_classpath() { - MVN="mvn" - if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn - fi - - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${PULSAR_HOME}/cli/target/classpath_shaded.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${PULSAR_HOME}/cli/pom.xml" dependency:build-classpath -DincludeScope=compile -Dmdep.outputFile="${f}" &> /dev/null - fi - PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -if [ -d "$PULSAR_HOME/lib" ]; then - PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*" -else - add_maven_deps_to_classpath -fi - -if [ -z "$PULSAR_CLIENT_CONF" ]; then - PULSAR_CLIENT_CONF=$DEFAULT_CLIENT_CONF -fi -if [ -z "$PULSAR_LOG_CONF" ]; then - PULSAR_LOG_CONF=$DEFAULT_LOG_CONF -fi - -PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH" -PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH" -OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`" -OPTS="$OPTS -Djava.net.preferIPv4Stack=true" - -OPTS="-cp $PULSAR_CLASSPATH $OPTS" - -OPTS="$OPTS $PULSAR_EXTRA_OPTS" - -# log directory & file -PULSAR_LOG_DIR=${PULSAR_LOG_DIR:-"$PULSAR_HOME/logs"} -PULSAR_LOG_APPENDER=${PULSAR_LOG_APPENDER:-"RoutingAppender"} -PULSAR_LOG_LEVEL=${PULSAR_LOG_LEVEL:-"info"} -PULSAR_ROUTING_APPENDER_DEFAULT=${PULSAR_ROUTING_APPENDER_DEFAULT:-"Console"} - -#Configure log configuration system properties -OPTS="$OPTS -Dpulsar.log.appender=$PULSAR_LOG_APPENDER" -OPTS="$OPTS -Dpulsar.log.dir=$PULSAR_LOG_DIR" -OPTS="$OPTS -Dpulsar.log.level=$PULSAR_LOG_LEVEL" -OPTS="$OPTS -Dpulsar.routing.appender.default=$PULSAR_ROUTING_APPENDER_DEFAULT" - -# find the java instance location -if [ ! -f "${JAVA_INSTANCE_JAR}" ]; then - # didn't find a released jar, then search the built jar - BUILT_JAVA_INSTANCE_JAR="${PULSAR_HOME}/runtime/target/java-instance.jar" - if [ -f "${BUILT_JAVA_INSTANCE_JAR}" ]; then - JAVA_INSTANCE_JAR=${BUILT_JAVA_INSTANCE_JAR} - else - echo "\nCouldn't find pulsar java instance jar."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi -fi - -# find the python instance location -if [ ! -f "${PY_INSTANCE_FILE}" ]; then - # didn't find a released python instance, then search the built python instance - BUILT_PY_INSTANCE_FILE="${PULSAR_HOME}/runtime/target/python-instance/python_instance_main.py" - if [ -f "${BUILT_PY_INSTANCE_FILE}" ]; then - PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE} - else - echo "\nCouldn't find pulsar python instance."; - echo "Make sure you've run 'mvn package'\n"; - exit 1; - fi -fi - -# functions -OPTS="$OPTS -Dpulsar.functions.java.instance.jar=${JAVA_INSTANCE_JAR}" -OPTS="$OPTS -Dpulsar.functions.python.instance.file=${PY_INSTANCE_FILE}" - -#Change to PULSAR_HOME to support relative paths -cd "$PULSAR_HOME" - -exec $JAVA $OPTS org.apache.pulsar.admin.cli.FunctionsTool $PULSAR_CLIENT_CONF "$@" diff --git a/pulsar-functions/conf/client.conf b/pulsar-functions/conf/client.conf deleted file mode 100644 index fabe68c3e011c..0000000000000 --- a/pulsar-functions/conf/client.conf +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Pulsar Client configuration -webServiceUrl=http://localhost:6750/ -brokerServiceUrl=pulsar://localhost:6650/ -#authPlugin= -#authParams= -#useTls= -#tlsAllowInsecureConnection -#tlsTrustCertsFilePath diff --git a/pulsar-functions/conf/function_worker.yml b/pulsar-functions/conf/function_worker.yml deleted file mode 100644 index 2bc51d1757c7c..0000000000000 --- a/pulsar-functions/conf/function_worker.yml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -workerId: standalone -workerHostname: localhost -workerPort: 6750 -functionMetadataTopicName: metadata -functionMetadataSnapshotsTopicPath: snapshots -clusterCoordinationTopicName: coordinate -pulsarFunctionsNamespace: sample/standalone/functions -pulsarServiceUrl: pulsar://localhost:6650 -pulsarWebServiceUrl: http://localhost:8080 -numFunctionPackageReplicas: 1 -downloadDirectory: /tmp/pulsar_functions -metricsConfig: - metricsSinkClassName: org.apache.pulsar.functions.metrics.sink.PrometheusSink - metricsCollectionInterval: 30 - metricsSinkConfig: - path: /metrics - port: 9099 -# threadContainerFactory: -# threadGroupName: "Thread Function Container Group" -processContainerFactory: - logDirectory: - -schedulerClassName: "org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler" -functionAssignmentTopicName: "assignments" -failureCheckFreqMs: 30000 -rescheduleTimeoutMs: 60000 -initialBrokerReconnectMaxRetries: 60 diff --git a/pulsar-functions/conf/log4j2.yml b/pulsar-functions/conf/log4j2.yml deleted file mode 100644 index 47b8dcb2372d2..0000000000000 --- a/pulsar-functions/conf/log4j2.yml +++ /dev/null @@ -1,167 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -Configuration: - name: pulsar-functions - monitorInterval: 30 - - Properties: - Property: - - name: "pulsar.log.dir" - value: "logs" - - name: "pulsar.log.file" - value: "pulsar-functions.log" - - name: "pulsar.log.appender" - value: "RoutingAppender" - - name: "pulsar.log.level" - value: "info" - - name: "pulsar.routing.appender.default" - value: "Console" - - name: "bk.log.level" - value: "info" - - Appenders: - - # Console - Console: - name: Console - target: SYSTEM_OUT - PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - - # Rolling file appender configuration - RollingFile: - name: RollingFile - fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}" - filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}-%d{MM-dd-yyyy}-%i.log.gz" - immediateFlush: true - PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - Policies: - TimeBasedTriggeringPolicy: - interval: 1 - modulate: true - SizeBasedTriggeringPolicy: - size: 1 GB - # Trigger every day at midnight that also scan - # roll-over strategy that deletes older file - CronTriggeringPolicy: - schedule: "0 0 0 * * ?" - # Delete file older than 30days - DefaultRolloverStrategy: - Delete: - basePath: ${sys:pulsar.log.dir} - maxDepth: 2 - IfFileName: - glob: "*/${sys:pulsar.log.file}*log.gz" - IfLastModified: - age: 30d - - # Rolling file appender configuration for bk - RollingRandomAccessFile: - name: BkRollingFile - fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk" - filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz" - immediateFlush: true - PatternLayout: - Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - Policies: - TimeBasedTriggeringPolicy: - interval: 1 - modulate: true - SizeBasedTriggeringPolicy: - size: 1 GB - # Trigger every day at midnight that also scan - # roll-over strategy that deletes older file - CronTriggeringPolicy: - schedule: "0 0 0 * * ?" - # Delete file older than 30days - DefaultRolloverStrategy: - Delete: - basePath: ${sys:pulsar.log.dir} - maxDepth: 2 - IfFileName: - glob: "*/${sys:pulsar.log.file}.bk*log.gz" - IfLastModified: - age: 30d - - # Routing - Routing: - name: RoutingAppender - Routes: - pattern: "$${ctx:function}" - Route: - - - Routing: - name: InstanceRoutingAppender - Routes: - pattern: "$${ctx:instance}" - Route: - - - RollingFile: - name: "Rolling-${ctx:function}" - fileName : "${sys:pulsar.log.dir}/functions/${ctx:function}/function.log" - filePattern : "${sys:pulsar.log.dir}/functions/${ctx:function}-%d{MM-dd-yyyy}-%i.log.gz" - PatternLayout: - Pattern: "%d{ABSOLUTE} %level{length=5} [%thread] [instance: %X{instance}] %logger{1} - %msg%n" - Policies: - TimeBasedTriggeringPolicy: - interval: 1 - modulate: true - SizeBasedTriggeringPolicy: - size: "20MB" - # Trigger every day at midnight that also scan - # roll-over strategy that deletes older file - CronTriggeringPolicy: - schedule: "0 0 0 * * ?" - # Delete file older than 30days - DefaultRolloverStrategy: - Delete: - basePath: ${sys:pulsar.log.dir} - maxDepth: 2 - IfFileName: - glob: "*/${sys:pulsar.log.file}*log.gz" - IfLastModified: - age: 30d - - ref: "${sys:pulsar.routing.appender.default}" - key: "${ctx:function}" - - ref: "${sys:pulsar.routing.appender.default}" - key: "${ctx:function}" - - Loggers: - - Logger: - name: org.apache.bookkeeper - level: "${sys:bk.log.level}" - additivity: false - AppenderRef: - - ref: BkRollingFile - - Logger: - name: org.apache.distributedlog - level: "${sys:bk.log.level}" - additivity: false - AppenderRef: - - ref: BkRollingFile - - Root: - level: info - AppenderRef: - - ref: "${sys:pulsar.log.appender}" - level: "${sys:pulsar.log.level}" diff --git a/pulsar-functions/conf/pulsar_env.sh b/pulsar-functions/conf/pulsar_env.sh deleted file mode 100644 index f600e6d20a389..0000000000000 --- a/pulsar-functions/conf/pulsar_env.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Set JAVA_HOME here to override the environment setting -# JAVA_HOME= - -# default settings for starting pulsar broker - -# Log4j configuration file -# PULSAR_LOG_CONF= - -# Logs location -# PULSAR_LOG_DIR= - -# Configuration file of settings used in broker server -# PULSAR_BROKER_CONF= - -# Configuration file of settings used in bookie server -# PULSAR_BOOKKEEPER_CONF= - -# Configuration file of settings used in zookeeper server -# PULSAR_ZK_CONF= - -# Configuration file of settings used in global zookeeper server -# PULSAR_GLOBAL_ZK_CONF= - -# Extra options to be passed to the jvm -PULSAR_MEM=" -Xms2g -Xmx2g -XX:MaxDirectMemorySize=4g" - -# Garbage collection options -PULSAR_GC=" -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB" - -# Extra options to be passed to the jvm -PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" - -# Add extra paths to the bookkeeper classpath -# PULSAR_EXTRA_CLASSPATH= - -#Folder where the Bookie server PID file should be stored -#PULSAR_PID_DIR= - -#Wait time before forcefully kill the pulser server instance, if the stop is not successful -#PULSAR_STOP_TIMEOUT= diff --git a/pulsar-functions/conf/pulsar_tools_env.sh b/pulsar-functions/conf/pulsar_tools_env.sh deleted file mode 100755 index 7eca6071d2480..0000000000000 --- a/pulsar-functions/conf/pulsar_tools_env.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Set JAVA_HOME here to override the environment setting -# JAVA_HOME= - -# default settings for starting pulsar broker - -# Log4j configuration file -# PULSAR_LOG_CONF= - -# Logs location -# PULSAR_LOG_DIR= - -# Configuration file of settings used in broker server -# PULSAR_BROKER_CONF= - -# Configuration file of settings used in bookie server -# PULSAR_BOOKKEEPER_CONF= - -# Configuration file of settings used in zookeeper server -# PULSAR_ZK_CONF= - -# Configuration file of settings used in global zookeeper server -# PULSAR_GLOBAL_ZK_CONF= - -# Extra options to be passed to the jvm -PULSAR_MEM=" -Xmx256m -XX:MaxDirectMemorySize=256m" - -# Garbage collection options -PULSAR_GC=" -client " - -# Extra options to be passed to the jvm -PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS} ${PULSAR_MEM} ${PULSAR_GC} -Dio.netty.leakDetectionLevel=disabled" - -# Add extra paths to the bookkeeper classpath -# PULSAR_EXTRA_CLASSPATH= - -#Folder where the Bookie server PID file should be stored -#PULSAR_PID_DIR= - -#Wait time before forcefully kill the pulser server instance, if the stop is not successful -#PULSAR_STOP_TIMEOUT= diff --git a/pulsar-functions/dist/pom.xml b/pulsar-functions/dist/pom.xml deleted file mode 100644 index 35066c23d5edf..0000000000000 --- a/pulsar-functions/dist/pom.xml +++ /dev/null @@ -1,103 +0,0 @@ - - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 2.0.0-incubating-SNAPSHOT - .. - - - pulsar-functions-dist - Pulsar Functions :: Distribution - pom - - - - org.apache.pulsar - pulsar-functions-worker - ${project.version} - - - - org.apache.pulsar - pulsar-functions-api-examples - ${project.version} - - provided - - - - io.netty - netty-all - - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - - - - - - maven-assembly-plugin - - - functions-dist-assembly - package - - single - - - true - posix - apache-pulsar-functions-${project.version} - - src/assemble/bin.xml - src/assemble/src.xml - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - - diff --git a/pulsar-functions/dist/src/assemble/bin.xml b/pulsar-functions/dist/src/assemble/bin.xml deleted file mode 100644 index b4def906e2e2b..0000000000000 --- a/pulsar-functions/dist/src/assemble/bin.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - bin - - tar.gz - - true - - - true - - - - - ${basedir}/../conf - - - ${basedir}/../bin - 755 - - - ${basedir}/../runtime/target/python-instance - instances/python-instance - - - - - ${basedir}/../runtime/target/java-instance.jar - java-instance.jar - instances - - - ${basedir}/../java-examples/target/pulsar-functions-api-examples.jar - api-examples.jar - examples - - - - - lib - false - compile - false - - ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} - - - - io.netty:netty-common - io.netty:netty-resolver - io.netty:netty-buffer - io.netty:netty-codec-http - io.netty:netty-codec - io.netty:netty-transport - io.netty:netty-handler - io.netty:netty-codec-http - - - - diff --git a/pulsar-functions/dist/src/assemble/src.xml b/pulsar-functions/dist/src/assemble/src.xml deleted file mode 100644 index 0c0526358e515..0000000000000 --- a/pulsar-functions/dist/src/assemble/src.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - src - - tar.gz - - true - - - .. - true - - **/README.md - **/DISCLAIMER - **/LICENSE - **/NOTICE - **/pom.xml - **/src/** - **/pulsar-client-cpp/** - **/conf/** - **/bin/** - **/*.txt - docker/** - dashboard/** - deployment/** - - - .git/** - **/.gitignore - **/.svn - **/*.iws - **/*.ipr - **/*.iml - **/*.cbp - **/*.pyc - **/.classpath - **/.project - **/.settings - **/target/** - **/CMakeFiles/** - **/CMakeCache.txt - **/cmake_install.cmake - pulsar-client-cpp/**/Makefile - pulsar-client-cpp/tests/main - pulsar-client-cpp/examples/SampleAsyncProducer - pulsar-client-cpp/examples/SampleConsumer - pulsar-client-cpp/examples/SampleConsumerListener - pulsar-client-cpp/examples/SampleProducer - pulsar-client-cpp/perf/perfProducer - pulsar-client-cpp/perf/perfConsumer - - **/python/dist/** - **/python/wheelhouse/** - **/python/MANIFEST - **/*.egg-info/** - - **/*.log - **/build/** - **/file:/** - **/SecurityAuth.audit* - **/site/** - **/.idea/** - **/*.a - **/*.so - **/*.so.* - **/*.dylib - - - - diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 5aacec56f179a..829f080ee0536 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -40,9 +40,7 @@ runtime worker worker-shaded - worker-runner java-examples - dist diff --git a/pulsar-functions/worker-runner/pom.xml b/pulsar-functions/worker-runner/pom.xml deleted file mode 100644 index 1e621d5cc6f58..0000000000000 --- a/pulsar-functions/worker-runner/pom.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 2.0.0-incubating-SNAPSHOT - .. - - - pulsar-functions-worker-runner - Pulsar Functions :: Worker Runner - - - - - ${project.groupId} - pulsar-functions-worker-shaded - ${project.parent.version} - - - com.google.protobuf - protobuf-lite - - - com.google.protobuf - protobuf-java - - - com.google.protobuf.nano - protobuf-javanano - - - com.google.protobuf - protobuf-java-util - - - io.grpc - grpc-protobuf - - - io.grpc - grpc-protobuf-lite - - - io.grpc - grpc-protobuf-nano - - - org.apache.pulsar - pulsar-functions-proto - - - org.apache.pulsar - pulsar-functions-utils - - - org.apache.pulsar - pulsar-functions-metrics - - - org.apache.pulsar - pulsar-functions-runtime - - - org.apache.pulsar - pulsar-functions-instance - - - org.apache.pulsar - pulsar-functions-worker - - - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - -