[Api draft][connector] Add Kudu source and sink connector

.dlc.json

@@ -17,6 +17,9 @@
     },
     {
       "pattern": "^/docs/category"
+    },
+    {
+      "pattern": "^https://opencollective.com"
     }
   ],
   "timeout": "10s",

.github/workflows/backend.yml

@@ -23,6 +23,7 @@ on:
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
+      - 'seatunnel-ui/**'
 
 concurrency:
   group: backend-${{ github.event.pull_request.number || github.ref }}
@@ -88,7 +89,7 @@ jobs:
         java: [ '8', '11' ]
         os: [ 'ubuntu-latest', 'windows-latest' ]
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 50
+    timeout-minutes: 80
     steps:
       - uses: actions/checkout@v3
         with:
@@ -114,7 +115,7 @@ jobs:
     name: Dependency licenses
     needs: [ sanity-check ]
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 40
     steps:
       - uses: actions/checkout@v3
         with:
@@ -154,7 +155,9 @@ jobs:
           cache: 'maven'
       - name: Run Unit tests
         run: |
-          ./mvnw -T 2C -B clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
+          ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
+        env:
+          MAVEN_OPTS: -Xmx2048m
 
   integration-test:
     name: Integration Test
@@ -175,4 +178,6 @@ jobs:
           cache: 'maven'
       - name: Run Integration tests
         run: |
-          ./mvnw -T 2C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
+          ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
+        env:
+          MAVEN_OPTS: -Xmx2048m

.github/workflows/code-analysys.yml

@@ -21,9 +21,11 @@ on:
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
+      - 'seatunnel-ui/**'
 jobs:
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 120
     steps:
       - uses: actions/checkout@v2
         with:

.github/workflows/codeql.yaml

@@ -22,12 +22,13 @@ on:
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
+      - 'seatunnel-ui/**'
 
 jobs:
   analyze:
     name: Analyze
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 60
     env:
       JAVA_TOOL_OPTIONS: -Xmx2G -Xms2G -Dhttp.keepAlive=false -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dlicense.skipAddThirdParty=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
 

.github/workflows/docker.yml

@@ -25,6 +25,7 @@ on:
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
+      - 'seatunnel-ui/**'
 
 concurrency:
   group: docker-${{ github.event.pull_request.number || github.ref }}
@@ -34,7 +35,7 @@ jobs:
   check:
     name: Spark
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 1.8

.github/workflows/license.yml

@@ -36,7 +36,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Check License Header
-        uses: apache/skywalking-eyes@main
+        uses: apache/skywalking-eyes/header@501a28d2fb4a9b962661987e50cf0219631b32ff
   auto-license:
     name: Auto License
     runs-on: ubuntu-latest

.github/workflows/publish-docker.yaml

@@ -24,6 +24,7 @@ on:
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
+      - 'seatunnel-ui/**'
 
 env:
   HUB: ghcr.io/${{ github.repository }}

.gitignore

@@ -13,6 +13,7 @@ target/
 # Intellij Idea files
 .idea/
 *.iml
+.idea/*
 
 .DS_Store
 
@@ -40,4 +41,4 @@ Test.scala
 test.conf
 log4j.properties
 spark-warehouse
-*.flattened-pom.xml
+*.flattened-pom.xml

docs/en/connector-v2/sink/File.mdx

@@ -0,0 +1,262 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# File
+
+## Description
+
+Output data to local or hdfs or s3 file.
+
+## Options
+
+<Tabs
+    groupId="engine-type"
+    defaultValue="LocalFile"
+    values={[
+        {label: 'LocalFile', value: 'LocalFile'},
+        {label: 'HdfsFile', value: 'HdfsFile'},
+    ]}>
+    <TabItem value="LocalFile">
+
+| name                              | type   | required | default value                                                 |
+| --------------------------------- | ------ | -------- | ------------------------------------------------------------- |
+| path                              | string | yes      | -                                                             |
+| file_name_expression              | string | no       | "${transactionId}"                                            |
+| file_format                       | string | no       | "text"                                                        |
+| filename_time_format              | string | no       | "yyyy.MM.dd"                                                  |
+| field_delimiter                   | string | no       | '\001'                                                        |
+| row_delimiter                     | string | no       | "\n"                                                          |
+| partition_by                      | array  | no       | -                                                             |
+| partition_dir_expression          | string | no       | "\${k0}=\${v0}\/\${k1}=\${v1}\/...\/\${kn}=\${vn}\/"          |
+| is_partition_field_write_in_file  | boolean| no       | false                                                         |
+| sink_columns                      | array  | no       | When this parameter is empty, all fields are sink columns     |
+| is_enable_transaction             | boolean| no       | true                                                          |
+| save_mode                         | string | no       | "error"                                                       |
+
+### path [string]
+
+The target dir path is required. The `hdfs file` starts with `hdfs://` , and the `local file` starts with `file://`,
+
+### file_name_expression [string]
+
+`file_name_expression` describes the file expression which will be created into the `path`. We can add the variable `${now}` or `${uuid}` in the `file_name_expression`, like `test_${uuid}_${now}`,
+`${now}` represents the current time, and its format can be defined by specifying the option `filename_time_format`.
+
+Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file.
+
+### file_format [string]
+
+We supported `file_format` is `text`.
+
+Please note that, The final file name will ends with the file_format's suffix, the suffix of the text file is `txt`.
+
+### filename_time_format [string]
+
+When the format in the `file_name_expression` parameter is `xxxx-${now}` , `filename_time_format` can specify the time format of the path, and the default value is `yyyy.MM.dd` . The commonly used time formats are listed as follows:
+
+| Symbol | Description        |
+| ------ | ------------------ |
+| y      | Year               |
+| M      | Month              |
+| d      | Day of month       |
+| H      | Hour in day (0-23) |
+| m      | Minute in hour     |
+| s      | Second in minute   |
+
+See [Java SimpleDateFormat](https://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html) for detailed time format syntax.
+
+### field_delimiter [string]
+
+The separator between columns in a row of data.
+
+### row_delimiter [string]
+
+The separator between rows in a file.
+
+### partition_by [array]
+
+Partition data based on selected fields
+
+### partition_dir_expression [string]
+
+If the `partition_by` is specified, we will generate the corresponding partition directory based on the partition information, and the final file will be placed in the partition directory.
+
+Default `partition_dir_expression` is `${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/`. `k0` is the first partition field and `v0` is the value of the first partition field.
+
+### is_partition_field_write_in_file [boolean]
+
+If `is_partition_field_write_in_file` is `true`, the partition field and the value of it will be write into data file.
+
+For example, if you want to write a Hive Data File, Its value should be `false`.
+
+### sink_columns [array]
+
+Which columns need be write to file, default value is all of the columns get from `Transform` or `Source`.
+The order of the fields determines the order in which the file is actually written.
+
+### is_enable_transaction [boolean]
+
+If `is_enable_transaction` is true, we will ensure that data will not be lost or duplicated when it is written to the target directory.
+
+Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file.
+
+Only support `true` now.
+
+### save_mode [string]
+
+Storage mode, currently supports `overwrite` , `append` , `ignore` and `error` . For the specific meaning of each mode, see [save-modes](https://spark.apache.org/docs/latest/sql-programming-guide.html#save-modes)
+
+Streaming Job not support `overwrite`.
+
+</TabItem>
+<TabItem value="HdfsFile">
+
+In order to use this connector, You must ensure your spark/flink cluster already integrated hadoop. The tested hadoop version is 2.x.
+
+| name                              | type   | required | default value                                                 |
+| --------------------------------- | ------ | -------- | ------------------------------------------------------------- |
+| path                              | string | yes      | -                                                             |
+| file_name_expression              | string | no       | "${transactionId}"                                            |
+| file_format                       | string | no       | "text"                                                        |
+| filename_time_format              | string | no       | "yyyy.MM.dd"                                                  |
+| field_delimiter                   | string | no       | '\001'                                                        |
+| row_delimiter                     | string | no       | "\n"                                                          |
+| partition_by                      | array  | no       | -                                                             |
+| partition_dir_expression          | string | no       | "\${k0}=\${v0}\/\${k1}=\${v1}\/...\/\${kn}=\${vn}\/"          |
+| is_partition_field_write_in_file  | boolean| no       | false                                                         |
+| sink_columns                      | array  | no       | When this parameter is empty, all fields are sink columns     |
+| is_enable_transaction             | boolean| no       | true                                                          |
+| save_mode                         | string | no       | "error"                                                       |
+
+### path [string]
+
+The target dir path is required. The `hdfs file` starts with `hdfs://` , and the `local file` starts with `file://`,
+
+### file_name_expression [string]
+
+`file_name_expression` describes the file expression which will be created into the `path`. We can add the variable `${now}` or `${uuid}` in the `file_name_expression`, like `test_${uuid}_${now}`,
+`${now}` represents the current time, and its format can be defined by specifying the option `filename_time_format`.
+
+Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file.
+
+### file_format [string]
+
+We supported `file_format` is `text`.
+
+Please note that, The final file name will ends with the file_format's suffix, the suffix of the text file is `txt`.
+
+### filename_time_format [string]
+
+When the format in the `file_name_expression` parameter is `xxxx-${now}` , `filename_time_format` can specify the time format of the path, and the default value is `yyyy.MM.dd` . The commonly used time formats are listed as follows:
+
+| Symbol | Description        |
+| ------ | ------------------ |
+| y      | Year               |
+| M      | Month              |
+| d      | Day of month       |
+| H      | Hour in day (0-23) |
+| m      | Minute in hour     |
+| s      | Second in minute   |
+
+See [Java SimpleDateFormat](https://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html) for detailed time format syntax.
+
+### field_delimiter [string]
+
+The separator between columns in a row of data.
+
+### row_delimiter [string]
+
+The separator between rows in a file.
+
+### partition_by [array]
+
+Partition data based on selected fields
+
+### partition_dir_expression [string]
+
+If the `partition_by` is specified, we will generate the corresponding partition directory based on the partition information, and the final file will be placed in the partition directory.
+
+Default `partition_dir_expression` is `${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/`. `k0` is the first partition field and `v0` is the value of the first partition field.
+
+### is_partition_field_write_in_file [boolean]
+
+If `is_partition_field_write_in_file` is `true`, the partition field and the value of it will be write into data file.
+
+For example, if you want to write a Hive Data File, Its value should be `false`.
+
+### sink_columns [array]
+
+Which columns need be write to file, default value is all of the columns get from `Transform` or `Source`.
+The order of the fields determines the order in which the file is actually written.
+
+### is_enable_transaction [boolean]
+
+If `is_enable_transaction` is true, we will ensure that data will not be lost or duplicated when it is written to the target directory.
+
+Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file.
+
+Only support `true` now.
+
+### save_mode [string]
+
+Storage mode, currently supports `overwrite` , `append` , `ignore` and `error` . For the specific meaning of each mode, see [save-modes](https://spark.apache.org/docs/latest/sql-programming-guide.html#save-modes)
+
+Streaming Job not support `overwrite`.
+</TabItem>
+</Tabs>
+
+## Example
+
+<Tabs
+    groupId="engine-type"
+    defaultValue="LocalFile"
+    values={[
+        {label: 'LocalFile', value: 'LocalFile'},
+        {label: 'HdfsFile', value: 'HdfsFile'},
+    ]}>
+<TabItem value="LocalFile">
+
+```bash
+
+LocalFile {
+    path="file:///tmp/hive/warehouse/test2"
+    field_delimiter="\t"
+    row_delimiter="\n"
+    partition_by=["age"]
+    partition_dir_expression="${k0}=${v0}"
+    is_partition_field_write_in_file=true
+    file_name_expression="${transactionId}_${now}"
+    file_format="text"
+    sink_columns=["name","age"]
+    filename_time_format="yyyy.MM.dd"
+    is_enable_transaction=true
+    save_mode="error"
+}
+
+```
+
+</TabItem>
+
+<TabItem value="LocalFile">
+
+```bash
+
+HdfsFile {
+    path="file:///tmp/hive/warehouse/test2"
+    field_delimiter="\t"
+    row_delimiter="\n"
+    partition_by=["age"]
+    partition_dir_expression="${k0}=${v0}"
+    is_partition_field_write_in_file=true
+    file_name_expression="${transactionId}_${now}"
+    file_format="text"
+    sink_columns=["name","age"]
+    filename_time_format="yyyy.MM.dd"
+    is_enable_transaction=true
+    save_mode="error"
+}
+
+```
+
+</TabItem>
+</Tabs>