fix [Feature][Connector] New SeaTunnel API Connectors apache#1946 Add…

… Druid Source&Sink
laglangyue · Jan 17, 2023 · 455012d · 455012d
1 parent 9df0094
commit 455012d
Show file tree

Hide file tree

Showing 17 changed files with 1,176 additions and 2 deletions.
diff --git a/docs/en/connector-v2/sink/Druid.md b/docs/en/connector-v2/sink/Druid.md
@@ -0,0 +1,101 @@
+# Druid
+
+> Druid sink connector
+
+## Description
+
+Write data to Apache Druid.
+
+
+## Options
+
+| name                    | type     | required | default value |
+| ----------------------- | -------- | -------- | ------------- |
+| coordinator_url         | `String` | yes      | -             |
+| datasource              | `String` | yes      | -             |
+| columns                 | `List<String>` | yes| __time        |
+| timestamp_column        | `String` | no       | timestamp     |
+| timestamp_format        | `String` | no       | auto          |
+| timestamp_missing_value | `String` | no       | -             |
+
+### coordinator_url [`String`]
+
+The URL of Coordinator service in Apache Druid.
+
+### datasource [`String`]
+
+The DataSource name in Apache Druid.
+
+### columns [`List<String>`]
+
+These columns that you want to write of Druid.
+
+### timestamp_column [`String`]
+
+The timestamp column name in Apache Druid, the default value is `timestamp`.
+
+### timestamp_format [`String`]
+
+The timestamp format in Apache Druid, the default value is `auto`, it could be:
+
+- `iso`
+  - ISO8601 with 'T' separator, like "2000-01-01T01:02:03.456"
+
+- `posix`
+  - seconds since epoch
+
+- `millis`
+  - milliseconds since epoch
+
+- `micro`
+  - microseconds since epoch
+
+- `nano`
+  - nanoseconds since epoch
+
+- `auto`
+  - automatically detects ISO (either 'T' or space separator) or millis format
+
+- any [Joda DateTimeFormat](http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html) string
+
+### timestamp_missing_value [`String`]
+
+The timestamp missing value in Apache Druid, which is used for input records that have a null or missing timestamp. The value of `timestamp_missing_value` should be in ISO 8601 format, for example `"2022-02-02T02:02:02.222"`.
+
+## Example
+
+### Simple
+
+```hocon
+DruidSink {
+  coordinator_url = "http://localhost:8081/"
+  datasource = "wikipedia"
+  columns = ["flags","page"]
+}
+```
+
+### Specified timestamp column and format
+
+```hocon
+DruidSink {
+  coordinator_url = "http://localhost:8081/"
+  datasource = "wikipedia"
+  timestamp_column = "timestamp"
+  timestamp_format = "auto"
+  columns = ["flags","page"]
+}
+```
+
+### Specified timestamp column, format and missing value
+
+```hocon
+DruidSink {
+  coordinator_url = "http://localhost:8081/"
+  datasource = "wikipedia"
+  timestamp_column = "timestamp"
+  timestamp_format = "auto"
+  timestamp_missing_value = "2022-02-02T02:02:02.222"
+  columns = ["flags","page"]
+}
+```
+
diff --git a/docs/en/connector-v2/source/Druid.md b/docs/en/connector-v2/source/Druid.md
@@ -0,0 +1,54 @@
+# Druid
+
+> Druid source connector
+
+## Description
+
+Read data from Apache Druid.
+
+## Options
+
+| name       | type           | required | default value |
+| ---------- | -------------- | -------- | ------------- |
+| url        | `String`       | yes      | -             |
+| datasource | `String`       | yes      | -             |
+| start_date | `String`       | no       | -             |
+| end_date   | `String`       | no       | -             |
+| columns    | `List<String>` | no       | `*`           |
+
+### url [`String`]
+
+The URL of JDBC of Apache Druid.
+
+### datasource [`String`]
+
+The DataSource name in Apache Druid.
+
+### start_date [`String`]
+
+The start date of DataSource, for example, `'2016-06-27'`, `'2016-06-27 00:00:00'`, etc.
+
+### end_date [`String`]
+
+The end date of DataSource, for example, `'2016-06-28'`, `'2016-06-28 00:00:00'`, etc.
+
+### columns [`List<String>`]
+
+These columns that you want to write  of DataSource.
+
+### common options [string]
+
+Source Plugin common parameters, refer to [Source Plugin](common-options.mdx) for details
+
+
+## Example
+
+```hocon
+DruidSource {
+  url = "jdbc:avatica:remote:url=http://localhost:8082/druid/v2/sql/avatica/"
+  datasource = "wikipedia"
+  start_date = "2016-06-27 00:00:00"
+  end_date = "2016-06-28 00:00:00"
+  columns = ["flags","page"]
+}
+```
diff --git a/plugin-mapping.properties b/plugin-mapping.properties
@@ -169,4 +169,5 @@ seatunnel.sink.S3Redshift = connector-s3-redshift
 seatunnel.source.TDengine = connector-tdengine
 seatunnel.sink.TDengine = connector-tdengine
 seatunnel.source.Persistiq = connector-http-persistiq
-
+seatunnel.source.Druid=connector-druid
+seatunnel.sink.Druid=connector-druid
diff --git a/pom.xml b/pom.xml
@@ -502,6 +502,43 @@
                 <scope>provided</scope>
             </dependency>
 
+
+            <dependency>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpclient</artifactId>
+                <version>${httpclient.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.alibaba</groupId>
+                <artifactId>fastjson</artifactId>
+                <version>${fastjson.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>joda-time</groupId>
+                <artifactId>joda-time</artifactId>
+                <version>${joda-tim.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.druid</groupId>
+                <artifactId>druid-indexing-service</artifactId>
+                <version>${druid.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.druid</groupId>
+                <artifactId>druid-core</artifactId>
+                <version>${druid-core.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.calcite.avatica</groupId>
+                <artifactId>avatica-core</artifactId>
+                <version>${avatica-core.version}</version>
+            </dependency>
+
         </dependencies>
 
     </dependencyManagement>
@@ -728,7 +765,7 @@
                         </execution>
                     </executions>
                 </plugin>
-                
+
                 <!-- checkstyle (End) -->
 
                 <plugin>

diff --git a/seatunnel-connectors-v2/connector-druid/pom.xml b/seatunnel-connectors-v2/connector-druid/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>seatunnel-connectors-v2</artifactId>
+        <groupId>org.apache.seatunnel</groupId>
+        <version>${revision}</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>connector-druid</artifactId>
+
+    <properties>
+        <jackson-datatype-joda.version>2.6.7</jackson-datatype-joda.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>connector-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-joda</artifactId>
+            <version>${jackson-datatype-joda.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.druid</groupId>
+            <artifactId>druid-indexing-service</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.druid</groupId>
+            <artifactId>druid-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.calcite.avatica</groupId>
+            <artifactId>avatica-core</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>