feat(java-binding): bundle jni library to jar (#10229)

.github/workflows/connector-node-integration.yml

@@ -41,7 +41,7 @@ jobs:
           
           echo "--- build connector node"
           cd ${RISINGWAVE_ROOT}/java
-          mvn --batch-mode --update-snapshots clean package
+          mvn --batch-mode --update-snapshots clean package -Dno-build-rust
           
           echo "--- install postgresql client"
           sudo apt install postgresql postgresql-contrib libpq-dev

ci/scripts/build.sh

@@ -52,13 +52,11 @@ cargo build \
     -p risingwave_sqlsmith \
     -p risingwave_compaction_test \
     -p risingwave_backup_cmd \
-    -p risingwave_java_binding \
     -p risingwave_e2e_extended_mode_test \
     $RISINGWAVE_FEATURE_FLAGS \
     --profile "$profile"
 
-# the file name suffix of artifact for risingwave_java_binding is so only for linux. It is dylib for MacOS
-artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test librisingwave_java_binding.so)
+artifacts=(risingwave sqlsmith compaction-test backup-restore risingwave_regress_test risingwave_e2e_extended_mode_test risedev-dev delete-range-test)
 
 echo "--- Show link info"
 ldd target/"$profile"/risingwave

ci/scripts/common.sh

@@ -86,23 +86,3 @@ function download_and_prepare_rw() {
   cargo make pre-start-dev
   cargo make --allow-private link-all-in-one-binaries
 }
-
-# Arguments:
-#   $1: cargo build `profile` of the binaries
-function download_java_binding() {
-  echo "--- Download java binding"
-  if [ -z "$1" ]; then
-    echo "download_java_binding: missing argument profile"
-    exit 1
-  fi
-
-  profile=$1
-
-  echo -e "\033[33mDownload artifacts\033[0m"
-
-  mkdir -p target/debug
-  download-and-decompress-artifact librisingwave_java_binding.so-"$profile" target/debug
-  mv target/debug/librisingwave_java_binding.so-"$profile" target/debug/librisingwave_java_binding.so
-
-  export RW_JAVA_BINDING_LIB_PATH=${PWD}/target/debug
-}

ci/scripts/e2e-iceberg-sink-test.sh

@@ -23,10 +23,6 @@ shift $((OPTIND -1))
 
 download_and_prepare_rw "$profile" source
 
-download_java_binding "$profile"
-
-export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk
-
 echo "--- Download connector node package"
 buildkite-agent artifact download risingwave-connector.tar.gz ./
 mkdir ./connector-node

ci/scripts/e2e-sink-test.sh

@@ -23,11 +23,6 @@ shift $((OPTIND -1))
 
 download_and_prepare_rw "$profile" source
 
-download_java_binding "$profile"
-
-# TODO: Switch to stream_chunk encoding once it's completed, and then remove json encoding as well as this env var.
-export RW_CONNECTOR_RPC_SINK_PAYLOAD_FORMAT=stream_chunk
-
 # Change process number limit
 echo "--- os limits"
 ulimit -a

ci/scripts/e2e-source-test.sh

@@ -26,8 +26,6 @@ shift $((OPTIND -1))
 
 download_and_prepare_rw "$profile" source
 
-download_java_binding "$profile"
-
 echo "--- Download connector node package"
 buildkite-agent artifact download risingwave-connector.tar.gz ./
 mkdir ./connector-node

ci/scripts/java-binding-test.sh

@@ -23,13 +23,11 @@ shift $((OPTIND -1))
 
 download_and_prepare_rw "$profile" source
 
-download_java_binding "$profile"
-
 echo "--- starting risingwave cluster"
 cargo make ci-start java-binding-demo
 
 echo "--- Build java binding demo"
-cargo make build-java-binding-java
+cargo make build-java-binding
 
 echo "--- ingest data and run java binding"
 cargo make ingest-data-and-run-java-binding

ci/scripts/release.sh

@@ -83,7 +83,7 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then
   gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz
 
   echo "--- Release build and upload risingwave connector node jar asset"
-  cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true
+  cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true
   cd connector-node/assembly/target && mv risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz
   gh release upload "${BUILDKITE_TAG}" risingwave-connector-"${BUILDKITE_TAG}".tar.gz
 fi

docker/Dockerfile

@@ -34,12 +34,11 @@ RUN rustup self update \
 
 RUN cargo fetch
 
-RUN cargo build -p risingwave_cmd_all -p risingwave_java_binding --release --features "rw-static-link" && \
+RUN cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \
   mkdir -p /risingwave/bin && mv /risingwave/target/release/risingwave /risingwave/bin/ && \
-  mkdir -p /risingwave/lib && mv /risingwave/target/release/librisingwave_java_binding.so /risingwave/lib && \
-  cargo clean
+  mkdir -p /risingwave/lib && cargo clean
 
-RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true && \
+RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \
     mkdir -p /risingwave/bin/connector-node && \
     tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
 
@@ -52,9 +51,6 @@ RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib
 COPY --from=builder /risingwave/bin/risingwave /risingwave/bin/risingwave
 COPY --from=builder /risingwave/bin/connector-node /risingwave/bin/connector-node
 COPY --from=builder /risingwave/ui /risingwave/ui
-COPY --from=builder /risingwave/lib/librisingwave_java_binding.so /risingwave/lib/librisingwave_java_binding.so
-# Set java.library.path env to /risingwave/lib
-ENV RW_JAVA_BINDING_LIB_PATH /risingwave/lib
 # Set default playground mode to docker-playground profile
 ENV PLAYGROUND_PROFILE docker-playground
 # Set default dashboard UI to local path instead of github proxy

docker/Dockerfile.hdfs

@@ -39,12 +39,11 @@ ARG JAVA_HOME_PATH
 ENV JAVA_HOME ${JAVA_HOME_PATH}
 ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH}
 
-RUN cargo build -p risingwave_cmd_all -p risingwave_java_binding --release --features "static-link static-log-level" && \
+RUN cargo build -p risingwave_cmd_all --release --features "static-link static-log-level" && \
   mkdir -p /risingwave/bin && mv /risingwave/target/release/risingwave /risingwave/bin/ && \
-  mkdir -p /risingwave/lib && mv /risingwave/target/release/librisingwave_java_binding.so /risingwave/lib && \
-  cargo clean
+  mkdir -p /risingwave/lib && cargo clean
 
-RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true && \
+RUN cd /risingwave/java && mvn -B package -Dmaven.test.skip=true -Djava.binding.release=true && \
     mkdir -p /risingwave/bin/connector-node && \
     tar -zxvf /risingwave/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz -C /risingwave/bin/connector-node
 
@@ -57,7 +56,6 @@ RUN mkdir -p /risingwave/bin/connector-node && mkdir -p /risingwave/lib
 COPY --from=builder /risingwave/bin/risingwave /risingwave/bin/risingwave
 COPY --from=builder /risingwave/bin/connector-node /risingwave/bin/connector-node
 COPY --from=builder /risingwave/ui /risingwave/ui
-COPY --from=builder /risingwave/lib/librisingwave_java_binding.so /risingwave/lib/librisingwave_java_binding.so
 
 # hadoop
 RUN wget https://rw-yufan.s3.ap-southeast-1.amazonaws.com/hadoop-2.7.3.tar.gz -P /root/
@@ -75,8 +73,6 @@ ARG JAVA_HOME_PATH
 ENV JAVA_HOME ${JAVA_HOME_PATH}
 ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH}
 
-# Set java.library.path env to /risingwave/lib
-ENV RW_JAVA_BINDING_LIB_PATH /risingwave/lib
 # Set default playground mode to docker-playground profile
 ENV PLAYGROUND_PROFILE docker-playground
 # Set default dashboard UI to local path instead of github proxy

java/connector-node/README.md

@@ -35,10 +35,7 @@ Sometimes, you need to specify the shared library path. For example, when progra
 ```
 RISINGWAVE_ROOT=$(git rev-parse --show-toplevel)
 CONNECTOR_ROOT=$JAVA_ROOT/connector-node
-# Build shared library file
-cd $RISINGWAVE_ROOT && cargo build -p risingwave_java_binding
-# specify the Djava.library.path, please make sure the shared library you needed exist in /target/debug
-cd ${CONNECTOR_ROOT}/assembly/target && java -classpath "./libs/*" -Djava.library.path=${RISINGWAVE_ROOT}/target/debug com.risingwave.connector.ConnectorService
+cd ${CONNECTOR_ROOT}/assembly/target && java -classpath "./libs/*" com.risingwave.connector.ConnectorService
 ```
 
 ## Docker image

java/connector-node/assembly/scripts/start-service.sh

@@ -25,4 +25,4 @@ if [ -z "${port}" ]; then
     port=$PORT
 fi
 
-java -classpath "${DIR}/libs/*" -Djava.library.path="${RW_JAVA_BINDING_LIB_PATH}" $MAIN --port ${port}
+java -classpath "${DIR}/libs/*" $MAIN --port ${port}

java/java-binding/pom.xml

@@ -11,15 +11,22 @@
     </parent>
 
     <artifactId>java-binding</artifactId>
-    
+
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>
         <module.version>1.0-SNAPSHOT</module.version>
+        <jni.loader.version>1.0.0</jni.loader.version>
+        <java.binding.release>false</java.binding.release>
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.questdb</groupId>
+            <artifactId>jar-jni</artifactId>
+            <version>${jni.loader.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.risingwave.java</groupId>
             <artifactId>proto</artifactId>
@@ -36,49 +43,54 @@
         </dependency>
     </dependencies>
 
+    <profiles>
+        <profile>
+            <id>no-build-jni-rust</id>
+            <activation>
+                <property>
+                    <name>no-build-rust</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.questdb</groupId>
+                        <artifactId>rust-maven-plugin</artifactId>
+                        <version>${jni.loader.version}</version>
+                        <executions>
+                            <execution>
+                                <id>build-jni-rust</id>
+                                <!--disable the plugin-->
+                                <phase>none</phase>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
     <build>
-        <pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
-            <plugins>
-                <!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
-                <plugin>
-                    <artifactId>maven-clean-plugin</artifactId>
-                    <version>3.1.0</version>
-                </plugin>
-                <!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
-                <plugin>
-                    <artifactId>maven-resources-plugin</artifactId>
-                    <version>3.0.2</version>
-                </plugin>
-                <plugin>
-                    <artifactId>maven-compiler-plugin</artifactId>
-                    <version>3.8.0</version>
-                </plugin>
-                <plugin>
-                    <artifactId>maven-surefire-plugin</artifactId>
-                    <version>2.22.1</version>
-                </plugin>
-                <plugin>
-                    <artifactId>maven-jar-plugin</artifactId>
-                    <version>3.0.2</version>
-                </plugin>
-                <plugin>
-                    <artifactId>maven-install-plugin</artifactId>
-                    <version>2.5.2</version>
-                </plugin>
-                <plugin>
-                    <artifactId>maven-deploy-plugin</artifactId>
-                    <version>2.8.2</version>
-                </plugin>
-                <!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
-                <plugin>
-                    <artifactId>maven-site-plugin</artifactId>
-                    <version>3.7.1</version>
-                </plugin>
-                <plugin>
-                    <artifactId>maven-project-info-reports-plugin</artifactId>
-                    <version>3.0.0</version>
-                </plugin>
-            </plugins>
-        </pluginManagement>
+        <plugins>
+            <plugin>
+                <groupId>org.questdb</groupId>
+                <artifactId>rust-maven-plugin</artifactId>
+                <version>${jni.loader.version}</version>
+                <executions>
+                    <execution>
+                        <id>build-jni-rust</id>
+                        <goals>
+                            <goal>build</goal>
+                        </goals>
+                        <configuration>
+                            <path>../../src/java_binding</path>
+                            <release>${java.binding.release}</release>
+                            <copyTo>${project.build.directory}/classes/risingwave/jni</copyTo>
+                            <copyWithPlatformDir>true</copyWithPlatformDir>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
     </build>
 </project>

java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java

@@ -14,9 +14,11 @@
 
 package com.risingwave.java.binding;
 
+import io.questdb.jar.jni.JarJniLoader;
+
 public class Binding {
     static {
-        System.loadLibrary("risingwave_java_binding");
+        JarJniLoader.loadLib(Binding.class, "/risingwave/jni", "risingwave_java_binding");
     }
 
     public static native int vnodeCount();

src/compute/src/server.rs

@@ -314,8 +314,8 @@ pub async fn compute_node_serve(
     let connector_params = risingwave_connector::ConnectorParams {
         connector_rpc_endpoint: opts.connector_rpc_endpoint,
         sink_payload_format: match opts.connector_rpc_sink_payload_format.as_deref() {
-            None | Some("json") => SinkPayloadFormat::Json,
-            Some("stream_chunk") => SinkPayloadFormat::StreamChunk,
+            None | Some("stream_chunk") => SinkPayloadFormat::StreamChunk,
+            Some("json") => SinkPayloadFormat::Json,
             _ => {
                 unreachable!(
                     "invalid sink payload format: {:?}. Should be either json or stream_chunk",

src/java_binding/Cargo.toml

@@ -37,7 +37,7 @@ tracing = "0.1"
 risingwave_expr = { path = "../expr" }
 
 [lib]
-crate_type = ["cdylib"]
+crate-type = ["cdylib"]
 
 [[bin]]
 name = "data-chunk-payload-generator"

src/java_binding/make-java-binding.toml

@@ -8,23 +8,14 @@ cd java
 javac -h . -cp java-binding/src/main/java/ java-binding/src/main/java/com/risingwave/java/binding/Binding.java
 '''
 
-[tasks.build-java-binding-rust]
-category = "RiseDev - Java Binding"
-description = "Build the java binding rust code"
-script = '''
-#!/usr/bin/env bash
-set -ex
-cargo build -p risingwave_java_binding
-'''
-
-[tasks.build-java-binding-java]
+[tasks.build-java-binding]
 category = "RiseDev - Java Binding"
 description = "Build the java binding java code"
 script = '''
 #!/usr/bin/env bash
 set -ex
 cd java
-mvn clean install --no-transfer-progress --pl java-binding-integration-test --am -DskipTests=true
+mvn install --no-transfer-progress --pl java-binding-integration-test --am -DskipTests=true
 mvn dependency:copy-dependencies --no-transfer-progress --pl java-binding-integration-test
 '''
 
@@ -78,8 +69,7 @@ bash ${RUST_JAVA_BINDING_ROOT}/run_demo.sh
 category = "RiseDev - Java Binding"
 description = "Run the java binding demo"
 dependencies = [
-    "build-java-binding-rust",
-    "build-java-binding-java",
+    "build-java-binding",
     "start-java-binding-demo-cluster",
     "ingest-data-and-run-java-binding",
     "kill-java-binding-demo-cluster"
@@ -89,8 +79,7 @@ dependencies = [
 category = "RiseDev - Java Binding"
 description = "Run the java binding stream chunk demo"
 dependencies = [
-    "build-java-binding-rust",
-    "build-java-binding-java",
+    "build-java-binding",
 ]
 script = '''
 #!/usr/bin/env bash
@@ -102,5 +91,5 @@ cd ${RISINGWAVE_ROOT}/java
 
 (cd ${RISINGWAVE_ROOT} && cargo run --bin data-chunk-payload-generator) | \
     java -cp "./java-binding-integration-test/target/dependency/*:./java-binding-integration-test/target/classes" \
-     -Djava.library.path=${RISINGWAVE_ROOT}/target/debug  com.risingwave.java.binding.StreamChunkDemo
+    com.risingwave.java.binding.StreamChunkDemo
 '''

src/java_binding/run_demo.sh

@@ -20,7 +20,7 @@ set -x
 cd ${RISINGWAVE_ROOT}/java
 
 java -cp "./java-binding-integration-test/target/dependency/*:./java-binding-integration-test/target/classes" \
-    -Djava.library.path=${RISINGWAVE_ROOT}/target/debug  com.risingwave.java.binding.HummockReadDemo
+    com.risingwave.java.binding.HummockReadDemo
 
 psql -d dev -h localhost -p 4566 -U root << EOF
 DROP TABLE ${TABLE_NAME};