Merge branch 'dev' into jdbc-sqlite

apache · Nov 15, 2022 · 70494ae · 70494ae
2 parents 18cf506 + bd9a313
commit 70494ae
Show file tree

Hide file tree

Showing 189 changed files with 8,844 additions and 588 deletions.
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -138,6 +138,8 @@ jobs:
               - "seatunnel-engine/**"
             engine-e2e:
               - "seatunnel-e2e/seatunnel-engine-e2e/**"
+            deleted-poms:
+              - deleted: "**/pom.xml" 
 
       - name: Check Connector V2 Update
         id: cv2-modules
@@ -193,13 +195,32 @@ jobs:
           echo $modules
           echo "modules=$modules" >> $GITHUB_OUTPUT
 
+      - name: Check Deleted Modules
+        id: deleted-modules
+        if: ${{ steps.filter.outputs.deleted-poms == 'true' }}
+        run: |
+          update_files='${{ steps.filter.outputs.deleted-poms_files }}'
+          modules=`python tools/update_modules_check/update_modules_check.py delete $update_files`
+          echo $modules
+          echo "modules=$modules" >> $GITHUB_OUTPUT      
+
       - name: Make unit test modules
         id: ut-modules
         if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '') }}
         run: |
           modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
           modules=${modules: 1}
           pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules`
+          # remove deleted modules
+          delete_modules='${{ steps.deleted-modules.outputs.modules }}'
+          if [[ "zz"$delete_modules != "zz" ]];then
+            pl_modules=`python tools/update_modules_check/update_modules_check.py rm $pl_modules $delete_modules`
+          fi
+          
+          if [[ "zz"$pl_modules == "zz" ]];then
+            exit 0
+          fi
+          
           ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt
           sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
           tree_modules="$modules$sub_modules"
@@ -216,6 +237,16 @@ jobs:
           modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
           modules=${modules: 1}
           pl_modules=`python tools/update_modules_check/update_modules_check.py replace $modules`
+          # remove deleted modules
+          delete_modules='${{ steps.deleted-modules.outputs.modules }}'
+          if [[ "zz"$delete_modules != "zz" ]];then
+            pl_modules=`python tools/update_modules_check/update_modules_check.py rm $pl_modules $delete_modules`
+          fi
+          
+          if [[ "zz"$pl_modules == "zz" ]];then
+            exit 0
+          fi
+          
           ./mvnw help:evaluate -Dexpression=project.modules -q -DforceStdout -pl $pl_modules > /tmp/sub_module.txt
           sub_modules=`python tools/update_modules_check/update_modules_check.py sub /tmp/sub_module.txt`
           tree_modules="$modules$sub_modules"

diff --git a/LICENSE b/LICENSE
@@ -213,7 +213,10 @@ seatunnel-api/src/main/java/org/apache/seatunnel/api/table/type/RowKind.java fro
 seatunnel-api/src/main/java/org/apache/seatunnel/api/state/CheckpointListener.java from https://github.com/apache/flink
 seatunnel-config/seatunnel-config-shade/src/main/java/org/apache/seatunnel/shade/com/typesafe/config/    from  https://github.com/lightbend/config
 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/  from https://github.com/apache/flink
+seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/reader/  from https://github.com/apache/flink
 seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/   from https://github.com/apache/iceberg
+seatunnel-connectors-v2/connector-cdc/connector-base/src/main/java/org/apache/seatunnel/connectors/cdc/base from https://github.com/ververica/flink-cdc-connectors
+seatunnel-connectors-v2/connector-cdc/connector-base/src/main/java/org/apache/seatunnel/connectors/cdc/debezium from https://github.com/ververica/flink-cdc-connectors
 generate_client_protocol.sh                                                                                                                 from https://github.com/hazelcast/hazelcast
 seatunnel-engine/seatunnel-engine-common/src/main/java/org/apache/seatunnel/engine/common/utils/ExceptionUtil.java                          from https://github.com/hazelcast/hazelcast
 seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/protocol/task/AbstractSeaTunnelMessageTask.java   from https://github.com/hazelcast/hazelcast

diff --git a/docs/en/connector-v2/sink/Kafka.md b/docs/en/connector-v2/sink/Kafka.md
@@ -25,7 +25,7 @@ By default, we will use 2pc to guarantee the message is sent to kafka exactly on
 | partition          | int                    | no       | -             |
 | assign_partitions  | list                   | no       | -             |
 | transaction_prefix | string                 | no       | -             |
-| common-options     |                        | no       | -             |
+| common-options     | config                 | no       | -             |
 
 ### topic [string]
 
@@ -88,7 +88,7 @@ This function by `MessageContentPartitioner` class implements `org.apache.kafka.
 If semantic is specified as EXACTLY_ONCE, the producer will write all messages in a Kafka transaction.
 Kafka distinguishes different transactions by different transactionId. This parameter is prefix of  kafka  transactionId, make sure different job use different prefix.
 
-### common options
+### common options [config]
 
 Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details.
 

diff --git a/docs/en/connector-v2/sink/Redis.md b/docs/en/connector-v2/sink/Redis.md
@@ -13,18 +13,18 @@ Used to write data to Redis.
 
 ##  Options
 
-| name           | type   | required | default value |
-|----------------|--------|----------|---------------|
-| host           | string | yes      | -             |
-| port           | int    | yes      | -             |
-| key            | string | yes      | -             |
-| data_type      | string | yes      | -             |
-| user           | string | no       | -             |
-| auth           | string | no       | -             |
-| mode           | string | no       | -             |
-| auth           | list   | no       | -             |
-| format         | string | no       | json          |
-| common-options |        | no       | -             |
+| name           | type   | required              | default value |
+|----------------|--------|-----------------------|--------------|
+| host           | string | yes                   | -            |
+| port           | int    | yes                   | -            |
+| key            | string | yes                   | -            |
+| data_type      | string | yes                   | -            |
+| user           | string | no                    | -            |
+| auth           | string | no                    | -            |
+| mode           | string | no                    | single       |
+| nodes          | list   | yes when mode=cluster | -            |
+| format         | string | no                    | json         |
+| common-options |        | no                    | -            |
 
 ### host [string]
 

diff --git a/docs/en/connector-v2/sink/Socket.md b/docs/en/connector-v2/sink/Socket.md
@@ -16,8 +16,8 @@ Used to send data to Socket Server. Both support streaming and batch mode.
 
 | name           | type   | required | default value |
 | -------------- |--------|----------|---------------|
-| host           | String | Yes      | -             |
-| port           | Integer| yes      | -             |
+| host           | String | Yes      |               |
+| port           | Integer| yes      |               |
 | max_retries    | Integer| No       | 3             |
 | common-options |        | no       | -             |
 

diff --git a/docs/en/connector-v2/sink/StarRocks.md b/docs/en/connector-v2/sink/StarRocks.md
@@ -18,7 +18,7 @@ The internal implementation of StarRocks sink connector is cached and imported b
 | username                    | string                       | yes      | -               |
 | password                    | string                       | yes      | -               |
 | database                    | string                       | yes      | -               |
-| table                       | string                       | no       | -               |
+| table                       | string                       | yes       | -               |
 | labelPrefix                 | string                       | no       | -               |
 | batch_max_rows              | long                         | no       | 1024            |
 | batch_max_bytes             | int                          | no       | 5 * 1024 * 1024 |
@@ -50,44 +50,46 @@ The name of StarRocks table
 
 ### labelPrefix [string]
 
-the prefix of  StarRocks stream load label
+The prefix of StarRocks stream load label
 
-### batch_max_rows [string]
+### batch_max_rows [long]
 
 For batch writing, when the number of buffers reaches the number of `batch_max_rows` or the byte size of `batch_max_bytes` or the time reaches `batch_interval_ms`, the data will be flushed into the StarRocks
 
-### batch_max_bytes [string]
+### batch_max_bytes [int]
 
 For batch writing, when the number of buffers reaches the number of `batch_max_rows` or the byte size of `batch_max_bytes` or the time reaches `batch_interval_ms`, the data will be flushed into the StarRocks
 
-### batch_interval_ms [string]
+### batch_interval_ms [int]
 
 For batch writing, when the number of buffers reaches the number of `batch_max_rows` or the byte size of `batch_max_bytes` or the time reaches `batch_interval_ms`, the data will be flushed into the StarRocks
 
-### max_retries [string]
+### max_retries [int]
 
 The number of retries to flush failed
 
-### retry_backoff_multiplier_ms [string]
+### retry_backoff_multiplier_ms [int]
 
 Using as a multiplier for generating the next delay for backoff
 
-### max_retry_backoff_ms [string]
+### max_retry_backoff_ms [int]
 
 The amount of time to wait before attempting to retry a request to `StarRocks`
 
 ### sink.properties.*  [starrocks stream load config]
 
-the parameter of the stream load `data_desc`
-The way to specify the parameter is to add the prefix `sink.properties.` to the original stream load parameter name. 
-For example, the way to specify `strip_outer_array` is: `sink.properties.strip_outer_array`.
+The parameter of the stream load `data_desc`
+The way to specify the parameter is to add the prefix `sink.properties.` to the original stream load parameter name 
+For example, the way to specify `strip_outer_array` is: `sink.properties.strip_outer_array`
 
 #### Supported import data formats
 
 The supported formats include CSV and JSON. Default value: CSV
 
 ## Example
+
 Use JSON format to import data
+
 ```
 sink {
     StarRocks {
@@ -105,6 +107,7 @@ sink {
 ```
 
 Use CSV format to import data
+
 ```
 sink {
     StarRocks {
@@ -115,8 +118,8 @@ sink {
         table = "e2e_table_sink"
         batch_max_rows = 10
         sink.properties.format = "CSV"
-        sink.properties.column_separator = "\\x01",
-        sink.properties.row_delimiter = "\\x02"
+        sink.properties.column_separator = "\x01"
+        sink.properties.row_delimiter = "\x02"
     }
 }
 ```

diff --git a/docs/en/connector-v2/sink/Tablestore.md b/docs/en/connector-v2/sink/Tablestore.md
@@ -0,0 +1,74 @@
+# Tablestore
+
+> Tablestore sink connector
+
+## Description
+
+Write data to `Tablestore`
+
+## Key features
+
+- [ ] [exactly-once](../../concept/connector-v2-features.md)
+- [ ] [schema projection](../../concept/connector-v2-features.md)
+
+## Options
+
+| name             | type   | required | default value |
+|----------------- | ------ |----------| ------------- |
+| end_point        | string | yes      | -             |
+| instance_name    | string | yes      | -             |
+| access_key_id    | string | yes      | -             |
+| access_key_secret| string | yes      | -             |
+| table            | string | yes      | -             |
+| primary_keys     | array  | yes      | -             |
+| batch_size       | string | no       | 25            |
+| batch_interval_ms| string | no       | 1000          |
+| common-options   | config | no       | -             |
+
+### end_point [string]
+
+endPoint to write to Tablestore.
+
+### instanceName [string]
+
+The instanceName of Tablestore.
+
+### access_key_id [string]
+
+The access id of Tablestore.
+
+### access_key_secret [string]
+
+The access secret of Tablestore.
+
+### table [string]
+
+The table of Tablestore.
+
+### primaryKeys [array]
+
+The primaryKeys of Tablestore.
+
+### common options [ config ]
+
+Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details.
+
+## Example
+
+```bash
+Tablestore {
+    end_point = "xxxx"
+    instance_name = "xxxx"
+    access_key_id = "xxxx"
+    access_key_secret = "xxxx"
+    table = "sink"
+    primary_keys = ["pk_1","pk_2","pk_3","pk_4"]
+  }
+```
+
+## Changelog
+
+### next version
+
+- Add Tablestore Sink Connector
+
diff --git a/docs/en/connector-v2/source/Jdbc.md b/docs/en/connector-v2/source/Jdbc.md
@@ -90,18 +90,19 @@ in parallel according to the concurrency of tasks.
 
 there are some reference value for params above.
 
-| datasource | driver                                       | url                                                                | maven                                                                                                       |
-|------------|----------------------------------------------|--------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------|
-| mysql      | com.mysql.cj.jdbc.Driver                     | jdbc:mysql://localhost:3306/test                                   | https://mvnrepository.com/artifact/mysql/mysql-connector-java                                               |
-| postgresql | org.postgresql.Driver                        | jdbc:postgresql://localhost:5432/postgres                          | https://mvnrepository.com/artifact/org.postgresql/postgresql                                                |
-| dm         | dm.jdbc.driver.DmDriver                      | jdbc:dm://localhost:5236                                           | https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18                                                |
-| phoenix    | org.apache.phoenix.queryserver.client.Driver | jdbc:phoenix:thin:url=http://localhost:8765;serialization=PROTOBUF | https://mvnrepository.com/artifact/com.aliyun.phoenix/ali-phoenix-shaded-thin-client                        |
-| sqlserver  | com.microsoft.sqlserver.jdbc.SQLServerDriver | jdbc:microsoft:sqlserver://localhost:1433                          | https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc                                       |
-| oracle     | oracle.jdbc.OracleDriver                     | jdbc:oracle:thin:@localhost:1521/xepdb1                            | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8                                          |
-| sqlite     | org.sqlite.JDBC                              | jdbc:sqlite:test.db                                                | https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc    |
-| gbase8a    | com.gbase.jdbc.Driver                        | jdbc:gbase://e2e_gbase8aDb:5258/test                               | https://www.gbase8.cn/wp-content/uploads/2020/10/gbase-connector-java-8.3.81.53-build55.5.7-bin_min_mix.jar |
-| starrocks  | com.mysql.cj.jdbc.Driver                     | jdbc:mysql://localhost:3306/test                                   | https://mvnrepository.com/artifact/mysql/mysql-connector-java                                               |
-| db2        | com.ibm.db2.jcc.DB2Driver                    | jdbc:db2://localhost:50000/testdb                                  | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc/db2jcc4                                           |
+| datasource | driver                                                                 | url                                                                                                  | maven                                                                                                       |
+|------------|------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------|
+| mysql      | com.mysql.cj.jdbc.Driver                                               | jdbc:mysql://localhost:3306/test                                                                     | https://mvnrepository.com/artifact/mysql/mysql-connector-java                                               |
+| postgresql | org.postgresql.Driver                                                  | jdbc:postgresql://localhost:5432/postgres                                                            | https://mvnrepository.com/artifact/org.postgresql/postgresql                                                |
+| dm         | dm.jdbc.driver.DmDriver                                                | jdbc:dm://localhost:5236                                                                             | https://mvnrepository.com/artifact/com.dameng/DmJdbcDriver18                                                |
+| phoenix    | org.apache.phoenix.queryserver.client.Driver                           | jdbc:phoenix:thin:url=http://localhost:8765;serialization=PROTOBUF                                   | https://mvnrepository.com/artifact/com.aliyun.phoenix/ali-phoenix-shaded-thin-client                        |
+| sqlserver  | com.microsoft.sqlserver.jdbc.SQLServerDriver                           | jdbc:microsoft:sqlserver://localhost:1433                                                            | https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc                                       |
+| oracle     | oracle.jdbc.OracleDriver                                               | jdbc:oracle:thin:@localhost:1521/xepdb1                                                              | https://mvnrepository.com/artifact/com.oracle.database.jdbc/ojdbc8                                          |
+| sqlite     | org.sqlite.JDBC                                                        | jdbc:sqlite:test.db                                                                                  | https://mvnrepository.com/artifact/org.xerial/sqlite-jdbc    |
+| gbase8a    | com.gbase.jdbc.Driver                                                  | jdbc:gbase://e2e_gbase8aDb:5258/test                                                                 | https://www.gbase8.cn/wp-content/uploads/2020/10/gbase-connector-java-8.3.81.53-build55.5.7-bin_min_mix.jar |
+| starrocks  | com.mysql.cj.jdbc.Driver                                               | jdbc:mysql://localhost:3306/test                                                                     | https://mvnrepository.com/artifact/mysql/mysql-connector-java                                               |
+| db2        | com.ibm.db2.jcc.DB2Driver                                              | jdbc:db2://localhost:50000/testdb                                                                    | https://mvnrepository.com/artifact/com.ibm.db2.jcc/db2jcc/db2jcc4                                           |
+| tablestore | com.alicloud.openservices.tablestore.jdbc.OTSDriver                    | "jdbc:ots:http s://myinstance.cn-hangzhou.ots.aliyuncs.com/myinstance"                                | https://mvnrepository.com/artifact/com.aliyun.openservices/tablestore-jdbc                                  |
 
 ## Example
 
@@ -151,3 +152,4 @@ parallel:
 
 - [BugFix] Fix jdbc split bug ([3220](https://github.com/apache/incubator-seatunnel/pull/3220))
 - [Feature] Support Sqlite JDBC Source ([3089](https://github.com/apache/incubator-seatunnel/pull/3089))
+- [Feature] Support Tablestore Source ([3309](https://github.com/apache/incubator-seatunnel/pull/3309))