[DSIP-38] Add plugin directory in binary package (#16613)

apache · Sep 23, 2024 · d67ea80 · d67ea80
1 parent 62d33a7
commit d67ea80
Show file tree

Hide file tree

Showing 221 changed files with 3,690 additions and 1,518 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -47,6 +47,7 @@ github:
           - Docs
           - Frontend Build
           - "E2E-K8S-Result"
+          - "API-Test-Result"
           - "Mergeable: milestone-label-check"
           - "Title Validator"
       required_pull_request_reviews:

diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml
@@ -51,6 +51,19 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Maximize runner space
+        uses: ./.github/actions/maximize-build-space
+        with:
+          root-reserve-mb: 30720
+          temp-reserve-mb: 10240
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
       - uses: actions/checkout@v4
         with:
           submodules: true
@@ -67,8 +80,8 @@ jobs:
       - name: Build Image
         run: |
           ./mvnw -B clean install \
-          -Dmaven.test.skip \
-          -Dmaven.javadoc.skip \
+          -Dmaven.test.skip=true \
+          -Dmaven.javadoc.skip=true \
           -Dspotless.skip=true \
           -Pdocker,release -Ddocker.tag=ci
       - name: Export Docker Images

diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -56,6 +56,19 @@ jobs:
         java: [ '8', '11' ]
     timeout-minutes: 30
     steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Maximize runner space
+        uses: ./.github/actions/maximize-build-space
+        with:
+          root-reserve-mb: 30720
+          temp-reserve-mb: 10240
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
       - uses: actions/checkout@v4
         with:
           submodules: true
@@ -78,10 +91,7 @@ jobs:
           ./mvnw -B clean install \
                  -Prelease \
                  -Dmaven.test.skip=true \
-                 -Dspotless.skip=true \
-                 -Dhttp.keepAlive=false \
-                 -Dmaven.wagon.http.pool=false \
-                 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
+                 -Dspotless.skip=true
       - name: Check dependency license
         run: tools/dependencies/check-LICENSE.sh
       - uses: actions/upload-artifact@v4

diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_mysql_registry/Dockerfile
@@ -22,22 +22,24 @@ RUN apt update ; \
 
 COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
 RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
 
 RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting dolphinscheduler_env.sh
 COPY .github/workflows/cluster-test/mysql_with_mysql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
 
 #Download mysql jar
-ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
-ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
-RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
+ENV MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+ENV MYSQL_DRIVER="mysql-connector-java-8.0.16.jar"
+RUN wget -O $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $MYSQL_URL
+RUN cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER
 
 #Deploy
 COPY .github/workflows/cluster-test/mysql_with_mysql_registry/deploy.sh /root/deploy.sh

diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/install_env.sh
diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/Dockerfile
@@ -22,22 +22,24 @@ RUN apt update ; \
 
 COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
 RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
 
 RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting dolphinscheduler_env.sh
 COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh
 
 #Download mysql jar
-ENV MYSQL_URL "https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
-ENV MYSQL_DRIVER "mysql-connector-java-8.0.16.jar"
-RUN wget -O $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $MYSQL_URL ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
-cp $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER
+ENV MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
+ENV MYSQL_DRIVER="mysql-connector-java-8.0.16.jar"
+RUN wget -O $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $MYSQL_URL
+RUN cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/tools/libs/$MYSQL_DRIVER; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/api-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/master-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/worker-server/libs/$MYSQL_DRIVER ; \
+    cp $DOLPHINSCHEDULER_HOME/libs/$MYSQL_DRIVER $DOLPHINSCHEDULER_HOME/alert-server/libs/$MYSQL_DRIVER
 
 #Deploy
 COPY .github/workflows/cluster-test/mysql_with_zookeeper_registry/deploy.sh /root/deploy.sh

diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/Dockerfile
@@ -22,10 +22,11 @@ RUN apt update ; \
 
 COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
 RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
 
 RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting dolphinscheduler_env.sh
 COPY .github/workflows/cluster-test/postgresql_with_postgresql_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh

diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/install_env.sh
diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/Dockerfile
@@ -22,10 +22,11 @@ RUN apt update ; \
 
 COPY ./apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz /root
 RUN tar -zxvf /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz -C ~
+RUN rm -f /root/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
 
 RUN mv /root/apache-dolphinscheduler-*-SNAPSHOT-bin /root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
-ENV DOLPHINSCHEDULER_HOME /root/apache-dolphinscheduler-test-SNAPSHOT-bin
+ENV DOLPHINSCHEDULER_HOME=/root/apache-dolphinscheduler-test-SNAPSHOT-bin
 
 #Setting dolphinscheduler_env.sh
 COPY .github/workflows/cluster-test/postgresql_with_zookeeper_registry/dolphinscheduler_env.sh $DOLPHINSCHEDULER_HOME/bin/env/dolphinscheduler_env.sh

diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/install_env.sh
diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml
@@ -49,26 +49,31 @@ jobs:
     needs: paths-filter
     if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 40
     steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Maximize runner space
+        uses: ./.github/actions/maximize-build-space
+        with:
+          root-reserve-mb: 30720
+          temp-reserve-mb: 10240
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
       - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Build Image
         run: |
           ./mvnw -B clean package \
-          -Dmaven.test.skip \
-          -Dmaven.javadoc.skip \
+          -Dmaven.test.skip=true \
+          -Dmaven.javadoc.skip=true \
           -Dspotless.skip=true \
-          -Dmaven.checkstyle.skip \
-          -Dmaven.deploy.skip \
-          -Ddocker.push.skip=true \
-          -Pdocker,release -Ddocker.tag=ci \
-          -pl org.apache.dolphinscheduler:dolphinscheduler-alert-server \
-          -pl dolphinscheduler-tools \
-          -pl dolphinscheduler-api \
-          -pl dolphinscheduler-master \
-          -pl dolphinscheduler-worker -am
+          -Pdocker,release -Ddocker.tag=ci
       - name: Create k8s Kind Cluster
         run: |
           # install kubectl