[Improve][Connector][Doris] Improve doris source. (#1359)

DTStack · Oct 27, 2022 · 0052d8d · 0052d8d
1 parent dde6452
commit 0052d8d
Show file tree

Hide file tree

Showing 30 changed files with 166 additions and 242 deletions.
diff --git a/...ris/src/main/java/com/dtstack/chunjun/connector/doris/converter/DorisColumnConverter.java b/...ris/src/main/java/com/dtstack/chunjun/connector/doris/converter/DorisColumnConverter.java
@@ -29,12 +29,6 @@
 import java.util.List;
 import java.util.StringJoiner;
 
-/**
- * Company：www.dtstack.com.
- *
- * @author shitou
- * @date 2021/11/10
- */
 public class DorisColumnConverter
         extends AbstractRowConverter<RowData, RowData, StringJoiner, String> {
 

diff --git a/...is/src/main/java/com/dtstack/chunjun/connector/doris/converter/DorisHttpRowConverter.java b/...is/src/main/java/com/dtstack/chunjun/connector/doris/converter/DorisHttpRowConverter.java
@@ -29,12 +29,6 @@
 
 import java.util.StringJoiner;
 
-/**
- * Company: www.dtstack.com
- *
- * @author xuchao
- * @date 2021-11-21
- */
 public class DorisHttpRowConverter
         extends AbstractRowConverter<RowData, RowData, StringJoiner, LogicalType> {
 

diff --git a/...is/src/main/java/com/dtstack/chunjun/connector/doris/converter/DorisRowTypeConverter.java b/...is/src/main/java/com/dtstack/chunjun/connector/doris/converter/DorisRowTypeConverter.java
@@ -25,10 +25,6 @@
 
 import java.util.Locale;
 
-/**
- * @author xuchao
- * @date 2021-11-21
- */
 public class DorisRowTypeConverter {
 
     public static DataType apply(String type) {
@@ -38,31 +34,41 @@ public static DataType apply(String type) {
                 return DataTypes.BOOLEAN();
             case "TINYINT":
                 return DataTypes.TINYINT();
+            case "TINYINT UNSIGNED":
             case "SMALLINT":
+                return DataTypes.SMALLINT();
+            case "SMALLINT UNSIGNED":
             case "MEDIUMINT":
+            case "MEDIUMINT UNSIGNED":
             case "INT":
             case "INTEGER":
             case "INT24":
                 return DataTypes.INT();
+            case "INT UNSIGNED":
             case "BIGINT":
                 return DataTypes.BIGINT();
+            case "BIGINT UNSIGNED":
+                return DataTypes.DECIMAL(20, 0);
             case "REAL":
             case "FLOAT":
+            case "FLOAT UNSIGNED":
                 return DataTypes.FLOAT();
             case "DECIMAL":
+            case "DECIMAL UNSIGNED":
             case "NUMERIC":
             case "DECIMALV2":
                 return DataTypes.DECIMAL(38, 18);
             case "DOUBLE":
+            case "DOUBLE UNSIGNED":
                 return DataTypes.DOUBLE();
             case "CHAR":
             case "VARCHAR":
             case "STRING":
-            case "JSON":
             case "TINYTEXT":
             case "TEXT":
             case "MEDIUMTEXT":
             case "LONGTEXT":
+            case "JSON":
             case "ENUM":
             case "SET":
                 return DataTypes.STRING();
@@ -81,8 +87,8 @@ public static DataType apply(String type) {
             case "LONGBLOB":
             case "BINARY":
             case "VARBINARY":
-                // BYTES 底层调用的是VARBINARY最大长度
             case "GEOMETRY":
+                // BYTES 底层调用的是VARBINARY最大长度
                 return DataTypes.BYTES();
             case "NULL_TYPE":
             case "NULL":

diff --git a/.../main/java/com/dtstack/chunjun/connector/doris/exception/DorisConnectFailedException.java b/.../main/java/com/dtstack/chunjun/connector/doris/exception/DorisConnectFailedException.java
@@ -18,10 +18,6 @@
 
 package com.dtstack.chunjun.connector.doris.exception;
 
-/**
- * @author tiezhu@dtstack
- * @date 2021/9/17 星期五
- */
 public class DorisConnectFailedException extends RuntimeException {
     public DorisConnectFailedException(String username, String hostUrl, Throwable cause) {
         super(String.format("User [%s] connect to [%s] failed.", username, hostUrl), cause);

diff --git a/...-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/DorisConf.java b/...-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/DorisConf.java
@@ -30,22 +30,12 @@
 import java.util.Objects;
 import java.util.Properties;
 
-/**
- * @author tiezhu@dtstack
- * @date 2021/9/16 星期四
- */
 public class DorisConf extends JdbcConf {
 
     private String database;
 
     private String table;
 
-    private String username;
-
-    private String password;
-
-    private String writeMode;
-
     private List<String> feNodes;
 
     private String url;
@@ -102,14 +92,6 @@ public void setPassword(String password) {
         this.password = password;
     }
 
-    public String getWriteMode() {
-        return writeMode;
-    }
-
-    public void setWriteMode(String writeMode) {
-        this.writeMode = writeMode;
-    }
-
     public List<String> getFeNodes() {
         return feNodes;
     }

diff --git a/...tor-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/DorisConfBuilder.java b/...tor-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/DorisConfBuilder.java
@@ -22,11 +22,8 @@
 import java.util.Properties;
 import java.util.StringJoiner;
 
-/**
- * @author tiezhu@dtstack
- * @date 2021/9/17 星期五
- */
 public class DorisConfBuilder {
+
     private final DorisConf dorisConf;
 
     public DorisConfBuilder() {
@@ -58,11 +55,6 @@ public DorisConfBuilder setPassword(String password) {
         return this;
     }
 
-    public DorisConfBuilder setWriteMode(String writeMode) {
-        this.dorisConf.setWriteMode(writeMode);
-        return this;
-    }
-
     public DorisConfBuilder setNameMapped(boolean needNameMapping) {
         this.dorisConf.setNameMapped(needNameMapping);
         return this;
@@ -101,10 +93,6 @@ public DorisConfBuilder setWaitRetryMills(long waitRetryMills) {
     public DorisConf build() {
         StringJoiner errorMessage = new StringJoiner("\n");
 
-        if (dorisConf.getFeNodes() == null || dorisConf.getFeNodes().isEmpty()) {
-            errorMessage.add("Doris FeNodes can not be empty!");
-        }
-
         if (dorisConf.getUsername() == null || dorisConf.getUsername().isEmpty()) {
             errorMessage.add("Doris Username can not be empty!");
         }

diff --git a/...-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/DorisKeys.java b/...-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/DorisKeys.java
@@ -18,10 +18,6 @@
 
 package com.dtstack.chunjun.connector.doris.options;
 
-/**
- * @author tiezhu@dtstack
- * @date 2021/9/17 星期五
- */
 public final class DorisKeys {
 
     public static final String FIELD_DELIMITER_KEY = "fieldDelimiter";

diff --git a/...nnector-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/DorisOptions.java b/...nnector-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/DorisOptions.java
@@ -23,12 +23,6 @@
 
 import java.util.List;
 
-/**
- * Company: www.dtstack.com
- *
- * @author xuchao
- * @date 2021-11-21
- */
 public class DorisOptions {
     public static final ConfigOption<List<String>> FENODES =
             ConfigOptions.key("feNodes")

diff --git a/...n-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/LoadConf.java b/...n-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/LoadConf.java
@@ -20,10 +20,6 @@
 
 import java.io.Serializable;
 
-/**
- * @author tiezhu@dtstack
- * @date 2021/9/18 星期六
- */
 public class LoadConf implements Serializable {
 
     private static final Long serialVersionUID = 1L;

diff --git a/...ctor-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/LoadConfBuilder.java b/...ctor-doris/src/main/java/com/dtstack/chunjun/connector/doris/options/LoadConfBuilder.java
@@ -18,10 +18,6 @@
 
 package com.dtstack.chunjun.connector.doris.options;
 
-/**
- * @author tiezhu@dtstack
- * @date 2021/9/18 星期六
- */
 public class LoadConfBuilder {
     private final LoadConf loadConf;
 

diff --git a/...unjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/Carrier.java b/...unjun-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/Carrier.java
@@ -30,12 +30,6 @@
 import java.util.StringJoiner;
 import java.util.stream.IntStream;
 
-/**
- * Company：www.dtstack.com.
- *
- * @author shitou
- * @date 2022/1/17
- */
 public class Carrier implements Serializable {
     private static final long serialVersionUID = 1L;
     private final List<Map<String, Object>> insertContent;

diff --git a/...nnector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/DorisLoadClient.java b/...nnector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/DorisLoadClient.java
@@ -50,12 +50,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-/**
- * Company：www.dtstack.com.
- *
- * @author shitou
- * @date 2021/12/21
- */
 public class DorisLoadClient implements Serializable {
     private static final long serialVersionUID = 1L;
     private static final Logger LOG = LoggerFactory.getLogger(DorisLoadClient.class);

diff --git a/...nnector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/DorisStreamLoad.java b/...nnector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/DorisStreamLoad.java
@@ -53,19 +53,15 @@
 import java.util.UUID;
 import java.util.stream.Collectors;
 
-/**
- * @author tiezhu@dtstack.com
- * @since 08/10/2021 Friday
- */
 public class DorisStreamLoad implements Serializable {
 
     private static final Logger LOG = LoggerFactory.getLogger(DorisStreamLoad.class);
     private static final ObjectMapper OM = new ObjectMapper();
     private static final List<String> DORIS_SUCCESS_STATUS =
             new ArrayList<>(Arrays.asList("Success", "Publish Timeout"));
     private static final String LOAD_URL_PATTERN = "http://%s/api/%s/%s/_stream_load?";
-    private String authEncoding;
-    private Properties streamLoadProp;
+    private final String authEncoding;
+    private final Properties streamLoadProp;
     private String hostPort;
     private DorisConf options;
 

diff --git a/...connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/FeRestService.java b/...connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/FeRestService.java
@@ -71,10 +71,6 @@
 import static com.dtstack.chunjun.connector.doris.options.DorisKeys.ILLEGAL_ARGUMENT_MESSAGE;
 import static com.dtstack.chunjun.connector.doris.options.DorisKeys.SHOULD_NOT_HAPPEN_MESSAGE;
 
-/**
- * @author tiezhu@dtstack
- * @date 2021/9/17 星期五
- */
 public class FeRestService implements Serializable {
 
     public static final int REST_RESPONSE_STATUS_OK = 200;

diff --git a/...onnector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/Backend.java b/...onnector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/Backend.java
@@ -23,10 +23,6 @@
 
 import java.util.List;
 
-/**
- * @author tiezhu@dtstack.com
- * @since 08/10/2021 Friday
- */
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class Backend {
 

diff --git a/...ector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/BackendRow.java b/...ector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/BackendRow.java
@@ -22,10 +22,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 
-/**
- * @author tiezhu@dtstack.com
- * @since 08/10/2021 Friday
- */
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class BackendRow {
 

diff --git a/...-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/Field.java b/...-connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/Field.java
@@ -22,10 +22,6 @@
 
 import java.util.Objects;
 
-/**
- * @author tiezhu@dtstack.com
- * @since 08/10/2021 Friday
- */
 public class Field {
     private String name;
     private String type;

diff --git a/...is/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/PartitionDefinition.java b/...is/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/PartitionDefinition.java
@@ -27,10 +27,6 @@
 import java.util.Set;
 import java.util.StringJoiner;
 
-/**
- * @author tiezhu@dtstack
- * @date 2021/9/18 星期六
- */
 public class PartitionDefinition implements Serializable, Comparable<PartitionDefinition> {
     private final String database;
     private final String table;

diff --git a/...nector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/QueryPlan.java b/...nector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/QueryPlan.java
@@ -21,10 +21,6 @@
 import java.util.Map;
 import java.util.Objects;
 
-/**
- * @author tiezhu@dtstack.com
- * @since 08/10/2021 Friday
- */
 public class QueryPlan {
     private int status;
     private String opaqued_query_plan;

diff --git a/...ctor-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/RespContent.java b/...ctor-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/RespContent.java
@@ -23,10 +23,6 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-/**
- * @author tiezhu@dtstack.com
- * @since 08/10/2021 Friday
- */
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class RespContent {
 

diff --git a/...connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/Schema.java b/...connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/Schema.java
@@ -24,10 +24,6 @@
 import java.util.List;
 import java.util.Objects;
 
-/**
- * @author tiezhu@dtstack.com
- * @since 08/10/2021 Friday
- */
 public class Schema {
     private int status = 0;
     private List<Field> properties;

diff --git a/...connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/Tablet.java b/...connector-doris/src/main/java/com/dtstack/chunjun/connector/doris/rest/module/Tablet.java
@@ -21,10 +21,6 @@
 import java.util.List;
 import java.util.Objects;
 
-/**
- * @author tiezhu@dtstack.com
- * @since 08/10/2021 Friday
- */
 public class Tablet {
     private List<String> routing;
     private int version;

diff --git a/...r-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisDynamicTableSink.java b/...r-doris/src/main/java/com/dtstack/chunjun/connector/doris/sink/DorisDynamicTableSink.java
@@ -41,12 +41,6 @@
 import java.util.Arrays;
 import java.util.List;
 
-/**
- * Company: www.dtstack.com
- *
- * @author xuchao
- * @date 2021-11-21
- */
 public class DorisDynamicTableSink extends JdbcDynamicTableSink {
 
     private final TableSchema physicalSchema;