WIP

README.md

@@ -21,7 +21,7 @@
 
 ![logo](https://pulsar.apache.org/img/pulsar.svg)
 
-[![docker pull](https://img.shields.io/docker/pulls/apachepulsar/pulsar-all.svg)](https://hub.docker.com/r/apachepulsar/pulsar)
+[![docker pull](https://img.shields.io/docker/pulls/apachepulsar/pulsar.svg)](https://hub.docker.com/r/apachepulsar/pulsar)
 [![contributors](https://img.shields.io/github/contributors-anon/apache/pulsar)](https://github.com/apache/pulsar/graphs/contributors)
 [![last commit](https://img.shields.io/github/last-commit/apache/pulsar)](https://github.com/apache/pulsar/commits/master)
 [![release](https://img.shields.io/github/v/release/apache/pulsar?sort=semver)](https://pulsar.apache.org/download/)
@@ -200,13 +200,13 @@ Here are some general instructions for building custom docker images:
 * Java 11 is the recommended JDK version in `branch-2.8`, `branch-2.9` and `branch-2.10`.
 * Java 17 is the recommended JDK version in `master`.
 
-The following command builds the docker images `apachepulsar/pulsar-all:latest` and `apachepulsar/pulsar:latest`:
+The following command builds the docker image `apachepulsar/pulsar:latest`:
 
 ```bash
 mvn clean install -DskipTests
 # setting DOCKER_CLI_EXPERIMENTAL=enabled is required in some environments with older docker versions
 export DOCKER_CLI_EXPERIMENTAL=enabled
-mvn package -Pdocker,-main -am -pl docker/pulsar-all -DskipTests
+mvn package -Pdocker,-main -am -pl docker/pulsar -DskipTests
 ```
 
 After the images are built, they can be tagged and pushed to your custom repository. Here's an example of a bash script that tags the docker images with the current version and git revision and pushes them to `localhost:32000/apachepulsar`.
@@ -217,8 +217,6 @@ pulsar_version=$(mvn initialize help:evaluate -Dexpression=project.version -pl .
 gitrev=$(git rev-parse HEAD | colrm 10)
 tag="${pulsar_version}-${gitrev}"
 echo "Using tag $tag"
-docker tag apachepulsar/pulsar-all:latest ${image_repo_and_project}/pulsar-all:$tag
-docker push ${image_repo_and_project}/pulsar-all:$tag
 docker tag apachepulsar/pulsar:latest ${image_repo_and_project}/pulsar:$tag
 docker push ${image_repo_and_project}/pulsar:$tag
 ```

build/run_unit_group.sh

@@ -167,7 +167,7 @@ function test_group_proxy() {
 
 function test_group_other() {
   mvn_test --clean --install \
-           -pl '!org.apache.pulsar:distribution,!org.apache.pulsar:pulsar-offloader-distribution,!org.apache.pulsar:pulsar-server-distribution,!org.apache.pulsar:pulsar-io-distribution,!org.apache.pulsar:pulsar-all-docker-image' \
+           -pl '!org.apache.pulsar:distribution,!org.apache.pulsar:pulsar-offloader-distribution,!org.apache.pulsar:pulsar-server-distribution,!org.apache.pulsar:pulsar-io-distribution,!org.apache.pulsar:pulsar-functions-python-docker-image' \
            -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DskipIntegrationTests \
            -Dexclude='**/ManagedLedgerTest.java,
                    **/OffloadersCacheTest.java

docker-compose/kitchen-sink/docker-compose.yml

@@ -28,7 +28,7 @@ services:
   zk1:
     container_name: zk1
     hostname: zk1
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: >
       bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \
                bin/apply-config-from-env.py conf/pulsar_env.sh && \
@@ -44,7 +44,7 @@ services:
   zk2:
     container_name: zk2
     hostname: zk2
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: >
       bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \
                bin/apply-config-from-env.py conf/pulsar_env.sh && \
@@ -60,7 +60,7 @@ services:
   zk3:
     container_name: zk3
     hostname: zk3
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: >
       bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \
                bin/apply-config-from-env.py conf/pulsar_env.sh && \
@@ -76,7 +76,7 @@ services:
   pulsar-init:
     container_name: pulsar-init
     hostname: pulsar-init
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: bin/init-cluster.sh
     environment:
       clusterName: test
@@ -95,7 +95,7 @@ services:
   bk1:
     hostname: bk1
     container_name: bk1
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: >
       bash -c "export dbStorage_writeCacheMaxSizeMb="$${dbStorage_writeCacheMaxSizeMb:-16}" && \
                export dbStorage_readAheadCacheMaxSizeMb="$${dbStorage_readAheadCacheMaxSizeMb:-16}" && \
@@ -121,7 +121,7 @@ services:
   bk2:
     hostname: bk2
     container_name: bk2
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: >
       bash -c "export dbStorage_writeCacheMaxSizeMb="${dbStorage_writeCacheMaxSizeMb:-16}" && \
                export dbStorage_readAheadCacheMaxSizeMb="${dbStorage_readAheadCacheMaxSizeMb:-16}" && \
@@ -148,7 +148,7 @@ services:
   bk3:
     hostname: bk3
     container_name: bk3
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: >
       bash -c "export dbStorage_writeCacheMaxSizeMb="${dbStorage_writeCacheMaxSizeMb:-16}" && \
                export dbStorage_readAheadCacheMaxSizeMb="${dbStorage_readAheadCacheMaxSizeMb:-16}" && \
@@ -176,7 +176,7 @@ services:
   broker1:
     hostname: broker1
     container_name: broker1
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     restart: on-failure
     command: >
       bash -c "bin/apply-config-from-env.py conf/broker.conf && \
@@ -205,7 +205,7 @@ services:
   broker2:
     hostname: broker2
     container_name: broker2
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     restart: on-failure
     command: >
       bash -c "bin/apply-config-from-env.py conf/broker.conf && \
@@ -235,7 +235,7 @@ services:
   broker3:
     hostname: broker3
     container_name: broker3
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     restart: on-failure
     command: >
       bash -c "bin/apply-config-from-env.py conf/broker.conf && \
@@ -267,7 +267,7 @@ services:
     hostname: proxy1
     container_name: proxy1
     restart: on-failure
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: >
       bash -c "bin/apply-config-from-env.py conf/proxy.conf && \
                bin/apply-config-from-env.py conf/pulsar_env.sh && \
@@ -300,7 +300,7 @@ services:
     hostname: websocket1
     container_name: websocket1
     restart: on-failure
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     command: >
       bash -c "bin/apply-config-from-env.py conf/websocket.conf && \
                bin/apply-config-from-env.py conf/pulsar_env.sh && \
@@ -328,7 +328,7 @@ services:
   fnc1:
     hostname: fnc1
     container_name: fnc1
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     restart: on-failure
     command: >
       bash -c "bin/apply-config-from-env.py conf/client.conf && \
@@ -364,7 +364,7 @@ services:
   sql1:
     hostname: sql1
     container_name: sql1
-    image: apachepulsar/pulsar-all:latest
+    image: apachepulsar/pulsar:latest
     restart: on-failure
     command: >
       bash -c "bin/apply-config-from-env.py conf/pulsar_env.sh && \

docker/README.md

@@ -24,7 +24,8 @@
 The Apache Pulsar community produces 2 docker images with each official release.
 
 * `apachepulsar/pulsar` - contains the necessary components for a working Pulsar cluster
-* `apachepulsar/pulsar-all` - extends the `apachepulsar/pulsar` image by adding many Pulsar connectors and offloaders
+* `apachepulsar/pulsar-functions-python` - extends the `apachepulsar/pulsar` image by adding the Python
+   dependencies required to run Pulsar Functions with Python runtime. 
 
 Since the 2.10.0 release, these docker images run as an unnamed, non-root user that is also part of the root group, by
 default. This was done to increase container security. The user is part of the root group to ensure that the container
@@ -37,44 +38,27 @@ Note that you first must build the project in order to have the right dependenci
 
 ## Building Derivative Custom Images
 
-If you find the `apachepulsar/pulsar-all` docker image too large, but you want to use a connector or an offloader,
-you can easily build an image with a curated list of connectors or offloaders based on the official Apache Pulsar
+You can easily build an image with a curated list of connectors or offloaders based on the official Apache Pulsar
 images. You can use the following sample docker image as a guide:
 
 ```Dockerfile
 ARG VERSION
 
-# Load the pulsar-all image as a builder image
-FROM apachepulsar/pulsar-all:${VERSION} as pulsar-all
-
 FROM apachepulsar/pulsar:${VERSION}
 
 # Add the cassandra connector
-COPY --from=pulsar-all /pulsar/connectors/pulsar-io-cassandra-*.nar /pulsar/connectors
-
-# Add the jcloud offloader
-COPY --from=pulsar-all /pulsar/connectors/tiered-storage-jcloud-*.nar /pulsar/offloaders
+RUN mkdir -p connectors && \
+    cd connectors && \
+    wget https://downloads.apache.org/pulsar/pulsar-${VERSION}/connectors/pulsar-io-cassandra-${VERSION}.nar 
 ```
 
-NOTE: the above example uses a wildcard in the `COPY` commands because argument expansion does not work for `COPY`.
-
 Assuming that you have the above `Dockerfile` in your local directory and are running docker on your local host, you can
-run the following command to build a custom image with the cassandra connector and the jcloud offloader.
+run the following command to build a custom image with the cassandra connector.
 
 ```shell
 docker build --build-arg VERSION=2.9.1 -t pulsar-custom:2.9.1 .
 ```
 
-For reference, here are the sizes of the official 2.9.1 docker images and the custom image built from the above
-`Dockerfile`:
-
-| REPOSITORY              | TAG   | SIZE   |
-| :---------------------- | :---- | :----- |
-| apachepulsar/pulsar     | 2.9.1 | 1.59GB |
-| apachepulsar/pulsar-all | 2.9.1 | 3.44GB |
-| pulsar-custom           | 2.9.1 | 1.6GB  |
-
-
 ## Troubleshooting non-root containers
 
 Troubleshooting is harder because the docker image runs as a non-root user. For example, a non-root user won't be able

docker/pom.xml

@@ -58,7 +58,7 @@
       </activation>
       <modules>
         <module>pulsar</module>
-        <module>pulsar-all</module>
+        <module>pulsar-functions-python</module>
       </modules>
       <build>
         <plugins>

docker/publish.sh

@@ -67,19 +67,19 @@ set -e
 
 if [[ "$GIT_BRANCH" == "master" ]]; then
   docker tag apachepulsar/pulsar:${IMAGE_TAG} ${docker_registry_org}/pulsar:latest
-  docker tag apachepulsar/pulsar-all:${IMAGE_TAG} ${docker_registry_org}/pulsar-all:latest
+  docker tag apachepulsar/pulsar-functions-python:${IMAGE_TAG} ${docker_registry_org}/pulsar-functions-python:latest
 fi
 
 docker tag apachepulsar/pulsar:${IMAGE_TAG} ${docker_registry_org}/pulsar:$MVN_VERSION
-docker tag apachepulsar/pulsar-all:${IMAGE_TAG} ${docker_registry_org}/pulsar-all:$MVN_VERSION
+docker tag apachepulsar/pulsar-functions-python:${IMAGE_TAG} ${docker_registry_org}/pulsar-functions-python:$MVN_VERSION
 
 # Push all images and tags
 if [[ "$GIT_BRANCH" == "master" ]]; then
   docker push ${docker_registry_org}/pulsar:latest
-  docker push ${docker_registry_org}/pulsar-all:latest
+  docker push ${docker_registry_org}/pulsar-functions-python:latest
 fi
 
 docker push ${docker_registry_org}/pulsar:$MVN_VERSION
-docker push ${docker_registry_org}/pulsar-all:$MVN_VERSION
+docker push ${docker_registry_org}/pulsar-functions-python:$MVN_VERSION
 
 echo "Finished pushing images to ${docker_registry_org}"

docker/pulsar-all/Dockerfile → docker/pulsar-functions-python/Dockerfile

@@ -17,16 +17,38 @@
 # under the License.
 #
 
+ARG PULSAR_CLIENT_PYTHON_VERSION
 ARG PULSAR_IMAGE
-FROM busybox as pulsar-all
 
-ARG PULSAR_IO_DIR
-ARG PULSAR_OFFLOADER_TARBALL
+## Create stage to build the Python dependencies
+## Since it needs to have GCC available, we're doing it in a different layer
+FROM alpine:3.19 AS python-deps
 
-ADD ${PULSAR_IO_DIR} /connectors
-ADD ${PULSAR_OFFLOADER_TARBALL} /
-RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders
+RUN apk add --no-cache \
+       bash \
+       python3-dev \
+       g++ \
+       musl-dev \
+       libffi-dev \
+       py3-pip \
+       py3-grpcio \
+       py3-yaml
 
+RUN pip3 install --break-system-packages \
+        kazoo
+
+ARG PULSAR_CLIENT_PYTHON_VERSION
+RUN pip3 install --break-system-packages \
+    pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION}
+
+# Main image stage
+ARG PULSAR_IMAGE
 FROM $PULSAR_IMAGE
-COPY --from=pulsar-all /connectors /pulsar/connectors
-COPY --from=pulsar-all /offloaders /pulsar/offloaders
+
+# Switch to root user to install PIP package
+USER root
+RUN apk add --no-cache py3-pip
+USER 10000
+
+# Copy Python depedencies from the other stage
+COPY --from=python-deps /usr/lib/python3.11/site-packages /usr/lib/python3.11/site-packages