fix [Feature][Connector-V2] Support Druid Source & Sink apache#2937 R…

…eview Code problem
guanboo · Sep 30, 2022 · 40412be · 40412be
1 parent 3b1a3d8
commit 40412be
Show file tree

Hide file tree

Showing 9 changed files with 179 additions and 115 deletions.
diff --git a/docs/en/connector-v2/sink/Druid.md b/docs/en/connector-v2/sink/Druid.md
@@ -4,37 +4,47 @@
 
 ## Description
 
-Write data to Apache Druid.
+Used to write data to Druid.
 
+## Key features
+
+- [ ] [exactly-once](../../concept/connector-v2-features.md)
+
+The Druid sink plug-in can achieve accuracy once by implementing idempotent writing, and needs to cooperate with aggregatingmergetree and other engines that support deduplication.
+
+- [ ] [schema projection](../../concept/connector-v2-features.md)
+
+
+:::
 
 ## Options
 
 | name                    | type     | required | default value |
 | ----------------------- | -------- | -------- | ------------- |
-| coordinator_url         | `String` | yes      | -             |
-| datasource              | `String` | yes      | -             |
-| columns                 | `List<String>` | yes| __time        |
-| timestamp_column        | `String` | no       | timestamp     |
-| timestamp_format        | `String` | no       | auto          |
-| timestamp_missing_value | `String` | no       | -             |
+| coordinator_url         | string | yes      | -             |
+| datasource              | string | yes      | -             |
+| columns                 | List<string> | yes| -             |
+| timestamp_column        | string | no       | timestamp     |
+| timestamp_format        | string | no       | auto          |
+| timestamp_missing_value | string| no       | -             |
 
-### coordinator_url [`String`]
+### coordinator_url [string]
 
-The URL of Coordinator service in Apache Druid.
+`Druid` cluster coordinator address, the format is `host:port` ,Such as `"host:8081"` .
 
-### datasource [`String`]
+### datasource [string]
 
-The DataSource name in Apache Druid.
+The `Druid` datasource
 
-### columns [`List<String>`]
+### columns [array]
 
-These columns that you want to write of Druid.
+The data field that needs to be output to `Druid` , if not configured, it will be automatically adapted according to the sink table `schema` .
 
-### timestamp_column [`String`]
+### timestamp_column [string]
 
 The timestamp column name in Apache Druid, the default value is `timestamp`.
 
-### timestamp_format [`String`]
+### timestamp_format [string]
 
 The timestamp format in Apache Druid, the default value is `auto`, it could be:
 
@@ -58,12 +68,10 @@ The timestamp format in Apache Druid, the default value is `auto`, it could be:
 
 - any [Joda DateTimeFormat](http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html) string
 
-### timestamp_missing_value [`String`]
+### timestamp_missing_value [string]
 
 The timestamp missing value in Apache Druid, which is used for input records that have a null or missing timestamp. The value of `timestamp_missing_value` should be in ISO 8601 format, for example `"2022-02-02T02:02:02.222"`.
 
-## Example
-
 ### Simple
 
 ```hocon
@@ -97,5 +105,4 @@ DruidSink {
   timestamp_missing_value = "2022-02-02T02:02:02.222"
   columns = ["flags","page"]
 }
-```
-
+```
diff --git a/docs/en/connector-v2/source/Druid.md b/docs/en/connector-v2/source/Druid.md
@@ -6,33 +6,44 @@
 
 Read data from Apache Druid.
 
+## Key features
+
+- [x] [batch](../../concept/connector-v2-features.md)
+- [ ] [stream](../../concept/connector-v2-features.md)
+- [ ] [exactly-once](../../concept/connector-v2-features.md)
+- [x] [schema projection](../../concept/connector-v2-features.md)
+
+:::tip
+
+Reading data from Druid can also be done using JDBC
+
 ## Options
 
 | name       | type           | required | default value |
 | ---------- | -------------- | -------- | ------------- |
-| url        | `String`       | yes      | -             |
-| datasource | `String`       | yes      | -             |
-| start_date | `String`       | no       | -             |
-| end_date   | `String`       | no       | -             |
-| columns    | `List<String>` | no       | `*`           |
+| url        | string       | yes      | -             |
+| datasource | string       | yes      | -             |
+| start_date | string       | no       | -             |
+| end_date   | string       | no       | -             |
+| columns    | List<string> | no       | *             |
 
-### url [`String`]
+### url [string]
 
-The URL of JDBC of Apache Druid.
+`Druid` cluster broker address, the format is `jdbc:avatica:remote:url=http://host:port/druid/v2/sql/avatica/` ,Such as `"jdbc:avatica:remote:url=http://localhost:8082/druid/v2/sql/avatica/"` .
 
-### datasource [`String`]
+### datasource [string]
 
 The DataSource name in Apache Druid.
 
-### start_date [`String`]
+### start_date [string]
 
 The start date of DataSource, for example, `'2016-06-27'`, `'2016-06-27 00:00:00'`, etc.
 
-### end_date [`String`]
+### end_date [string]
 
 The end date of DataSource, for example, `'2016-06-28'`, `'2016-06-28 00:00:00'`, etc.
 
-### columns [`List<String>`]
+### columns [List<string>]
 
 These columns that you want to write  of DataSource.
 
@@ -43,6 +54,15 @@ Source Plugin common parameters, refer to [Source Plugin](common-options.mdx) fo
 
 ## Example
 
+```hocon
+DruidSource {
+  url = "jdbc:avatica:remote:url=http://localhost:8082/druid/v2/sql/avatica/"
+  datasource = "wikipedia"
+  start_date = "2016-06-27 00:00:00"
+  end_date = "2016-06-28 00:00:00"
+}
+```
+
 ```hocon
 DruidSource {
   url = "jdbc:avatica:remote:url=http://localhost:8082/druid/v2/sql/avatica/"

diff --git a/pom.xml b/pom.xml
@@ -89,6 +89,7 @@
         <module>seatunnel-formats</module>
         <module>seatunnel-dist</module>
 	    <module>seatunnel-server</module>
+        <module>seatunnel-engine</module>
     </modules>
 
     <profiles>
@@ -143,9 +144,6 @@
         <jackson.version>2.12.6</jackson.version>
         <lombok.version>1.18.0</lombok.version>
         <commons-compress.version>1.20</commons-compress.version>
-        <mysql.version>8.0.16</mysql.version>
-        <postgresql.version>42.3.3</postgresql.version>
-        <dm-jdbc.version>8.1.2.141</dm-jdbc.version>
         <skip.pmd.check>false</skip.pmd.check>
         <maven.deploy.skip>false</maven.deploy.skip>
         <maven.javadoc.skip>false</maven.javadoc.skip>
@@ -164,8 +162,11 @@
         <elasticsearch7.client.version>7.5.1</elasticsearch7.client.version>
         <flink-shaded-hadoop-2.version>2.7.5-7.0</flink-shaded-hadoop-2.version>
         <commons-lang3.version>3.4</commons-lang3.version>
+        <commons-io.version>2.11.0</commons-io.version>
         <commons-collections4.version>4.4</commons-collections4.version>
         <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
+        <commons-compress.version>1.20</commons-compress.version>
+        <protostuff.version>1.8.0</protostuff.version>
         <spark.scope>provided</spark.scope>
         <flink.scope>provided</flink.scope>
         <codec.version>1.13</codec.version>
@@ -184,7 +185,7 @@
         <docker-maven-plugin.version>0.38.0</docker-maven-plugin.version>
         <p3c-pmd.version>1.3.0</p3c-pmd.version>
         <maven-scm-provider-jgit.version>1.9.5</maven-scm-provider-jgit.version>
-        <testcontainer.version>1.16.3</testcontainer.version>
+        <testcontainer.version>1.17.3</testcontainer.version>
         <!-- Option args -->
         <skipUT>false</skipUT>
         <skipIT>true</skipIT>
@@ -198,8 +199,6 @@
         <snappy-java.version>1.1.8.3</snappy-java.version>
         <checker.qual.version>3.10.0</checker.qual.version>
         <awaitility.version>4.2.0</awaitility.version>
-        <httpclient.version>4.5.13</httpclient.version>
-        <fastjson.version>1.2.83</fastjson.version>
         <avatica-core.version>1.17.0</avatica-core.version>
         <druid-core.version>0.23.0</druid-core.version>
         <joda-tim.version>2.10.14</joda-tim.version>
@@ -281,6 +280,12 @@
                 <version>${jackson.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-compress</artifactId>
+                <version>${commons-compress.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.testcontainers</groupId>
                 <artifactId>testcontainers</artifactId>
@@ -298,12 +303,6 @@
                 </exclusions>
             </dependency>
 
-            <dependency>
-                <groupId>org.apache.commons</groupId>
-                <artifactId>commons-compress</artifactId>
-                <version>${commons-compress.version}</version>
-            </dependency>
-
             <dependency>
                 <groupId>org.apache.logging.log4j</groupId>
                 <artifactId>log4j-core</artifactId>
@@ -326,6 +325,7 @@
                 <artifactId>guava</artifactId>
                 <version>${guava.version}</version>
             </dependency>
+
             <dependency>
                 <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-api</artifactId>
@@ -358,15 +358,28 @@
             </dependency>
 
             <dependency>
-                <groupId>org.apache.httpcomponents</groupId>
-                <artifactId>httpclient</artifactId>
-                <version>${httpclient.version}</version>
+                <groupId>commons-io</groupId>
+                <artifactId>commons-io</artifactId>
+                <version>${commons-io.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>io.protostuff</groupId>
+                <artifactId>protostuff-core</artifactId>
+                <version>${protostuff.version}</version>
             </dependency>
 
             <dependency>
-                <groupId>com.alibaba</groupId>
-                <artifactId>fastjson</artifactId>
-                <version>${fastjson.version}</version>
+                <groupId>io.protostuff</groupId>
+                <artifactId>protostuff-runtime</artifactId>
+                <version>${protostuff.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.google.auto.service</groupId>
+                <artifactId>auto-service</artifactId>
+                <version>${auto-service.version}</version>
+                <scope>provided</scope>
             </dependency>
 
             <dependency>

diff --git a/seatunnel-connectors-v2/connector-druid/pom.xml b/seatunnel-connectors-v2/connector-druid/pom.xml
@@ -1,4 +1,18 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
 <project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -13,6 +27,7 @@
 
     <properties>
         <jackson-datatype-joda.version>2.6.7</jackson-datatype-joda.version>
+        <httpclient.version>4.5.13</httpclient.version>
     </properties>
 
     <dependencies>
@@ -30,10 +45,6 @@
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.alibaba</groupId>
-            <artifactId>fastjson</artifactId>
-        </dependency>
         <dependency>
             <groupId>joda-time</groupId>
             <artifactId>joda-time</artifactId>
@@ -50,6 +61,11 @@
             <groupId>org.apache.calcite.avatica</groupId>
             <artifactId>avatica-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>${httpclient.version}</version>
+        </dependency>
     </dependencies>
 
-</project>
+</project>