From 18f525e6cb7a504d88ce38a8ec2850a68b7c2c92 Mon Sep 17 00:00:00 2001 From: chaorongzhi Date: Fri, 8 Dec 2023 14:17:00 +0800 Subject: [PATCH] =?UTF-8?q?feat(hw):=E8=A7=A3=E5=86=B3=20=E4=B8=A4?= =?UTF-8?q?=E4=B8=AA=E5=8C=85=E7=9A=84=E5=85=BC=E5=AE=B9=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connector-elasticsearch-hw/pom.xml | 105 ---- .../hw/catalog/ElasticSearchCatalog.java | 222 -------- .../catalog/ElasticSearchCatalogFactory.java | 44 -- .../ElasticSearchDataTypeConvertor.java | 122 ----- .../elasticsearch/hw/client/EsRestClient.java | 472 ------------------ .../hw/config/EsClusterConnectionConfig.java | 83 --- .../elasticsearch/hw/config/SinkConfig.java | 65 --- .../elasticsearch/hw/config/SourceConfig.java | 64 --- .../hw/constant/ElasticsearchVersion.java | 60 --- .../constant/EsTypeMappingSeaTunnelType.java | 65 --- .../elasticsearch/hw/dto/BulkResponse.java | 58 --- .../hw/dto/ElasticsearchClusterInfo.java | 41 -- .../elasticsearch/hw/dto/IndexInfo.java | 51 -- .../hw/dto/source/IndexDocsCount.java | 41 -- .../hw/dto/source/ScrollResult.java | 30 -- .../hw/dto/source/SourceIndexInfo.java | 35 -- .../ElasticsearchConnectorErrorCode.java | 51 -- .../ElasticsearchConnectorException.java | 37 -- .../serialize/ElasticsearchRowSerializer.java | 174 ------- .../hw/serialize/KeyExtractor.java | 98 ---- .../hw/serialize/SeaTunnelRowSerializer.java | 25 - .../hw/serialize/index/IndexSerializer.java | 26 - .../index/IndexSerializerFactory.java | 38 -- .../index/impl/FixedValueIndexSerializer.java | 36 -- .../index/impl/VariableIndexSerializer.java | 68 --- .../DefaultSeaTunnelRowDeserializer.java | 233 --------- .../serialize/source/ElasticsearchRecord.java | 33 -- .../source/SeaTunnelRowDeserializer.java | 25 - .../serialize/type/IndexTypeSerializer.java | 25 - .../type/IndexTypeSerializerFactory.java | 52 -- .../type/impl/NotIndexTypeSerializer.java | 29 -- .../impl/RequiredIndexTypeSerializer.java | 37 -- .../hw/sink/ElasticsearchSink.java | 76 --- .../hw/sink/ElasticsearchSinkFactory.java | 55 -- .../hw/sink/ElasticsearchSinkWriter.java | 137 ----- .../hw/source/ElasticsearchSource.java | 120 ----- .../hw/source/ElasticsearchSourceFactory.java | 61 --- .../hw/source/ElasticsearchSourceReader.java | 132 ----- .../hw/source/ElasticsearchSourceSplit.java | 41 -- .../ElasticsearchSourceSplitEnumerator.java | 220 -------- .../hw/source/ElasticsearchSourceState.java | 32 -- .../ElasticsearchAggregatedCommitInfo.java | 26 - .../hw/state/ElasticsearchCommitInfo.java | 34 -- .../hw/state/ElasticsearchSinkState.java | 22 - .../elasticsearch/hw/util/RegexUtils.java | 37 -- .../elasticsearch/hw/util/SSLUtils.java | 165 ------ .../main/resources/conf/esParams.properties | 12 - .../src/main/resources/conf/krb5.conf | 47 -- .../ElasticsearchFactoryTest.java | 344 ------------- .../hive/HiveJdbcConnectionProvider.java | 13 +- .../seatunnel/jdbc/utils/LoginUtil.java | 461 +++++++++++++++++ .../connector-kafka-hw/pom.xml | 98 ---- .../seatunnel/kafka/hw/config/Config.java | 180 ------- .../kafka/hw/config/KafkaSemantics.java | 38 -- .../kafka/hw/config/MessageFormat.java | 28 -- .../config/MessageFormatErrorHandleWay.java | 23 - .../seatunnel/kafka/hw/config/StartMode.java | 45 -- .../hw/exception/KafkaConnectorErrorCode.java | 56 --- .../hw/exception/KafkaConnectorException.java | 36 -- .../DefaultSeaTunnelRowSerializer.java | 236 --------- .../hw/serialize/SeaTunnelRowSerializer.java | 33 -- .../kafka/hw/sink/KafkaInternalProducer.java | 181 ------- .../hw/sink/KafkaNoTransactionSender.java | 105 ---- .../kafka/hw/sink/KafkaProduceSender.java | 61 --- .../seatunnel/kafka/hw/sink/KafkaSink.java | 84 ---- .../kafka/hw/sink/KafkaSinkCommitter.java | 93 ---- .../kafka/hw/sink/KafkaSinkFactory.java | 68 --- .../kafka/hw/sink/KafkaSinkWriter.java | 239 --------- .../kafka/hw/sink/KafkaTransactionSender.java | 133 ----- .../hw/sink/MessageContentPartitioner.java | 62 --- .../kafka/hw/source/ConsumerMetadata.java | 43 -- .../kafka/hw/source/KafkaConsumerThread.java | 109 ---- .../kafka/hw/source/KafkaSource.java | 99 ---- .../kafka/hw/source/KafkaSourceConfig.java | 244 --------- .../kafka/hw/source/KafkaSourceFactory.java | 72 --- .../kafka/hw/source/KafkaSourceReader.java | 281 ----------- .../kafka/hw/source/KafkaSourceSplit.java | 92 ---- .../hw/source/KafkaSourceSplitEnumerator.java | 381 -------------- .../hw/state/KafkaAggregatedCommitInfo.java | 30 -- .../kafka/hw/state/KafkaCommitInfo.java | 34 -- .../kafka/hw/state/KafkaSinkState.java | 34 -- .../kafka/hw/state/KafkaSourceState.java | 40 -- .../kafka/hw/utils/KafkaSASLConstants.java | 32 -- .../seatunnel/kafka/hw/utils/LoginUtil.java | 169 ------- .../seatunnel/kafka/KafkaFactoryTest.java | 33 -- .../seatunnel/kafka/KafkaStartOffsetTest.java | 113 ----- seatunnel-connectors-v2/pom.xml | 8 +- .../seatunnel-engine-examples/pom.xml | 12 +- .../engine/SeaTunnelEngineExample.java | 2 +- 89 files changed, 489 insertions(+), 7718 deletions(-) delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/pom.xml delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchCatalog.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchCatalogFactory.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchDataTypeConvertor.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/client/EsRestClient.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/EsClusterConnectionConfig.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/SinkConfig.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/SourceConfig.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/constant/ElasticsearchVersion.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/constant/EsTypeMappingSeaTunnelType.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/BulkResponse.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/ElasticsearchClusterInfo.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/IndexInfo.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/IndexDocsCount.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/ScrollResult.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/SourceIndexInfo.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/exception/ElasticsearchConnectorErrorCode.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/exception/ElasticsearchConnectorException.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/ElasticsearchRowSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/KeyExtractor.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/SeaTunnelRowSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/IndexSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/IndexSerializerFactory.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/impl/FixedValueIndexSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/impl/VariableIndexSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/DefaultSeaTunnelRowDeserializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/ElasticsearchRecord.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/SeaTunnelRowDeserializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/IndexTypeSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/IndexTypeSerializerFactory.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/impl/NotIndexTypeSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/impl/RequiredIndexTypeSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSink.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSinkFactory.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSinkWriter.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSource.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceFactory.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceReader.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceSplit.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceSplitEnumerator.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceState.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchAggregatedCommitInfo.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchCommitInfo.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchSinkState.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/util/RegexUtils.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/util/SSLUtils.java delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/resources/conf/esParams.properties delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/resources/conf/krb5.conf delete mode 100644 seatunnel-connectors-v2/connector-elasticsearch-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/ElasticsearchFactoryTest.java create mode 100644 seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/LoginUtil.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/pom.xml delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/Config.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/KafkaSemantics.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/MessageFormat.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/MessageFormatErrorHandleWay.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/StartMode.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/exception/KafkaConnectorErrorCode.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/exception/KafkaConnectorException.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/serialize/DefaultSeaTunnelRowSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/serialize/SeaTunnelRowSerializer.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaInternalProducer.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaNoTransactionSender.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaProduceSender.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSink.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkCommitter.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkFactory.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkWriter.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaTransactionSender.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/MessageContentPartitioner.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/ConsumerMetadata.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaConsumerThread.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSource.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceConfig.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceFactory.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceReader.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceSplit.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceSplitEnumerator.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaAggregatedCommitInfo.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaCommitInfo.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaSinkState.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaSourceState.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/utils/KafkaSASLConstants.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/utils/LoginUtil.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/KafkaFactoryTest.java delete mode 100644 seatunnel-connectors-v2/connector-kafka-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/KafkaStartOffsetTest.java diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/pom.xml b/seatunnel-connectors-v2/connector-elasticsearch-hw/pom.xml deleted file mode 100644 index 13feda22f74..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/pom.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - 4.0.0 - - org.apache.seatunnel - seatunnel-connectors-v2 - ${revision} - - - connector-elasticsearch-hw - SeaTunnel : Connectors V2 : Elasticsearch-hw - - - 7.5.1 - 31.1-jre - - - - - org.elasticsearch.client - elasticsearch-rest-client-huawei - 6.7.1 - - - org.apache.httpcomponents - httpasyncclient - 4.1.4 - - - org.apache.seatunnel - connector-common - ${project.version} - compile - - - org.apache.seatunnel - seatunnel-format-json - ${project.version} - - - io.airlift - security - 206 - - - com.google.guava - guava - - - - - - com.google.guava - guava - ${guava.version} - - - org.apache.httpcomponents - httpclient - 4.5.14 - compile - - - - - - libs-release - http://192.168.12.88:8082/artifactory/libs-release-local/ - - - libs-snapshot - http://192.168.12.88:8082/artifactory/libs-snapshot-local/ - - - - true - - - true - - huaweicloudsdk - https://mirrors.huaweicloud.com/repository/maven/huaweicloudsdk/ - - - - diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchCatalog.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchCatalog.java deleted file mode 100644 index ca96c319af9..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchCatalog.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.catalog; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.apache.seatunnel.api.configuration.util.ConfigUtil; -import org.apache.seatunnel.api.table.catalog.Catalog; -import org.apache.seatunnel.api.table.catalog.CatalogTable; -import org.apache.seatunnel.api.table.catalog.PhysicalColumn; -import org.apache.seatunnel.api.table.catalog.TableIdentifier; -import org.apache.seatunnel.api.table.catalog.TablePath; -import org.apache.seatunnel.api.table.catalog.TableSchema; -import org.apache.seatunnel.api.table.catalog.exception.CatalogException; -import org.apache.seatunnel.api.table.catalog.exception.DatabaseAlreadyExistException; -import org.apache.seatunnel.api.table.catalog.exception.DatabaseNotExistException; -import org.apache.seatunnel.api.table.catalog.exception.TableAlreadyExistException; -import org.apache.seatunnel.api.table.catalog.exception.TableNotExistException; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.client.EsRestClient; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.ElasticsearchClusterInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source.IndexDocsCount; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Elasticsearch catalog implementation. - * - *

In ElasticSearch, we use the index as the database and table. - */ -public class ElasticSearchCatalog implements Catalog { - - private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchCatalog.class); - - private final String catalogName; - private final String defaultDatabase; - private final Config pluginConfig; - - private EsRestClient esRestClient; - - // todo: do we need default database? - public ElasticSearchCatalog( - String catalogName, String defaultDatabase, Config elasticSearchConfig) { - this.catalogName = checkNotNull(catalogName, "catalogName cannot be null"); - this.defaultDatabase = defaultDatabase; - this.pluginConfig = checkNotNull(elasticSearchConfig, "elasticSearchConfig cannot be null"); - } - - @Override - public void open() throws CatalogException { - try { - esRestClient = EsRestClient.createInstance(pluginConfig); - ElasticsearchClusterInfo elasticsearchClusterInfo = esRestClient.getClusterInfo(); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - "Success open es catalog: {}, cluster info: {}", - catalogName, - elasticsearchClusterInfo); - } - } catch (Exception e) { - throw new CatalogException(String.format("Failed to open catalog %s", catalogName), e); - } - } - - @Override - public void close() throws CatalogException { - esRestClient.close(); - } - - @Override - public String getDefaultDatabase() throws CatalogException { - return defaultDatabase; - } - - @Override - public boolean databaseExists(String databaseName) throws CatalogException { - // check if the index exist - try { - List indexDocsCount = esRestClient.getIndexDocsCount(databaseName); - return true; - } catch (Exception e) { - throw new CatalogException( - String.format( - "Failed to check if catalog %s database %s exists", - catalogName, databaseName), - e); - } - } - - @Override - public List listDatabases() throws CatalogException { - return esRestClient.listIndex(); - } - - @Override - public List listTables(String databaseName) - throws CatalogException, DatabaseNotExistException { - if (!databaseExists(databaseName)) { - throw new DatabaseNotExistException(catalogName, databaseName); - } - return Lists.newArrayList(databaseName); - } - - @Override - public boolean tableExists(TablePath tablePath) throws CatalogException { - checkNotNull(tablePath); - // todo: Check if the database name is the same with table name - return databaseExists(tablePath.getTableName()); - } - - @Override - public CatalogTable getTable(TablePath tablePath) - throws CatalogException, TableNotExistException { - // Get the index mapping? - checkNotNull(tablePath, "tablePath cannot be null"); - ElasticSearchDataTypeConvertor elasticSearchDataTypeConvertor = - new ElasticSearchDataTypeConvertor(); - TableSchema.Builder builder = TableSchema.builder(); - Map fieldTypeMapping = - esRestClient.getFieldTypeMapping(tablePath.getTableName(), Collections.emptyList()); - fieldTypeMapping.forEach( - (fieldName, fieldType) -> { - // todo: we need to add a new type TEXT or add length in STRING type - PhysicalColumn physicalColumn = - PhysicalColumn.of( - fieldName, - elasticSearchDataTypeConvertor.toSeaTunnelType( - fieldName, fieldType), - null, - true, - null, - null); - builder.column(physicalColumn); - }); - - return CatalogTable.of( - TableIdentifier.of( - catalogName, tablePath.getDatabaseName(), tablePath.getTableName()), - builder.build(), - buildTableOptions(tablePath), - Collections.emptyList(), - ""); - } - - @Override - public void createTable(TablePath tablePath, CatalogTable table, boolean ignoreIfExists) - throws TableAlreadyExistException, DatabaseNotExistException, CatalogException { - // Create the index - checkNotNull(tablePath, "tablePath cannot be null"); - if (tableExists(tablePath)) { - if (ignoreIfExists) { - return; - } else { - throw new TableAlreadyExistException(catalogName, tablePath, null); - } - } - esRestClient.createIndex(tablePath.getTableName()); - } - - @Override - public void dropTable(TablePath tablePath, boolean ignoreIfNotExists) - throws TableNotExistException, CatalogException { - checkNotNull(tablePath); - if (!tableExists(tablePath) && !ignoreIfNotExists) { - throw new TableNotExistException(catalogName, tablePath); - } - try { - esRestClient.dropIndex(tablePath.getTableName()); - } catch (Exception ex) { - throw new CatalogException( - String.format( - "Failed to drop table %s in catalog %s", - tablePath.getTableName(), catalogName), - ex); - } - } - - @Override - public void createDatabase(TablePath tablePath, boolean ignoreIfExists) - throws DatabaseAlreadyExistException, CatalogException { - createTable(tablePath, null, ignoreIfExists); - } - - @Override - public void dropDatabase(TablePath tablePath, boolean ignoreIfNotExists) - throws DatabaseNotExistException, CatalogException { - dropTable(tablePath, ignoreIfNotExists); - } - - private Map buildTableOptions(TablePath tablePath) { - Map options = new HashMap<>(); - options.put("connector", "elasticsearch"); - // todo: Right now, we don't use the config in the plugin config, do we need to add - // bootstrapt servers here? - options.put("config", ConfigUtil.convertToJsonString(tablePath)); - return options; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchCatalogFactory.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchCatalogFactory.java deleted file mode 100644 index bfb237fea3a..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchCatalogFactory.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.catalog; - -import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.table.catalog.Catalog; -import org.apache.seatunnel.api.table.factory.CatalogFactory; - -public class ElasticSearchCatalogFactory implements CatalogFactory { - - @Override - public Catalog createCatalog(String catalogName, ReadonlyConfig options) { - // todo: - return null; - } - - @Override - public String factoryIdentifier() { - // todo: - return "Elasticsearch"; - } - - @Override - public OptionRule optionRule() { - // todo: - return null; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchDataTypeConvertor.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchDataTypeConvertor.java deleted file mode 100644 index a3eb8171407..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/catalog/ElasticSearchDataTypeConvertor.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.catalog; - -import org.apache.seatunnel.api.table.catalog.DataTypeConvertor; -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.LocalTimeType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SqlType; - -import com.google.auto.service.AutoService; - -import java.util.Map; - -import static com.google.common.base.Preconditions.checkNotNull; - -@AutoService(DataTypeConvertor.class) -public class ElasticSearchDataTypeConvertor implements DataTypeConvertor { - - public static final String STRING = "string"; - public static final String KEYWORD = "keyword"; - public static final String TEXT = "text"; - public static final String BOOLEAN = "boolean"; - public static final String BYTE = "byte"; - public static final String SHORT = "short"; - public static final String INTEGER = "integer"; - public static final String LONG = "long"; - public static final String FLOAT = "float"; - public static final String HALF_FLOAT = "half_float"; - public static final String DOUBLE = "double"; - public static final String DATE = "date"; - - @Override - public SeaTunnelDataType toSeaTunnelType(String field, String connectorDataType) { - return toSeaTunnelType(field, connectorDataType, null); - } - - @Override - public SeaTunnelDataType toSeaTunnelType( - String field, String connectorDataType, Map dataTypeProperties) { - checkNotNull(connectorDataType, "connectorDataType can not be null"); - switch (connectorDataType) { - case STRING: - return BasicType.STRING_TYPE; - case KEYWORD: - return BasicType.STRING_TYPE; - case TEXT: - return BasicType.STRING_TYPE; - case BOOLEAN: - return BasicType.BOOLEAN_TYPE; - case BYTE: - return BasicType.BYTE_TYPE; - case SHORT: - return BasicType.SHORT_TYPE; - case INTEGER: - return BasicType.INT_TYPE; - case LONG: - return BasicType.LONG_TYPE; - case FLOAT: - return BasicType.FLOAT_TYPE; - case HALF_FLOAT: - return BasicType.FLOAT_TYPE; - case DOUBLE: - return BasicType.DOUBLE_TYPE; - case DATE: - return LocalTimeType.LOCAL_DATE_TIME_TYPE; - default: - return BasicType.STRING_TYPE; - } - } - - @Override - public String toConnectorType( - String field, - SeaTunnelDataType seaTunnelDataType, - Map dataTypeProperties) { - checkNotNull(seaTunnelDataType, "seaTunnelDataType can not be null"); - SqlType sqlType = seaTunnelDataType.getSqlType(); - switch (sqlType) { - case STRING: - return STRING; - case BOOLEAN: - return BOOLEAN; - case BYTES: - return BYTE; - case TINYINT: - return SHORT; - case INT: - return INTEGER; - case BIGINT: - return LONG; - case FLOAT: - return FLOAT; - case DOUBLE: - return DOUBLE; - case TIMESTAMP: - return DATE; - default: - return STRING; - } - } - - @Override - public String getIdentity() { - return "Elasticsearch"; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/client/EsRestClient.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/client/EsRestClient.java deleted file mode 100644 index 4ba4ecabc99..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/client/EsRestClient.java +++ /dev/null @@ -1,472 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.client; - -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.TextNode; -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.apache.seatunnel.common.utils.JsonUtils; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.EsClusterConnectionConfig; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.BulkResponse; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.ElasticsearchClusterInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source.IndexDocsCount; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source.ScrollResult; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorException; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.http.HttpStatus; -import org.apache.http.util.Asserts; -import org.apache.http.util.EntityUtils; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.hwclient.HwRestClient; - -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -public class EsRestClient { - - private static final int CONNECTION_REQUEST_TIMEOUT = 10 * 1000; - - private static final int SOCKET_TIMEOUT = 5 * 60 * 1000; - - private final RestClient restClient; - - private EsRestClient(RestClient restClient) { - this.restClient = restClient; - } - - public static EsRestClient createInstance(Config pluginConfig) { - List hosts = pluginConfig.getStringList(EsClusterConnectionConfig.HOSTS.key()); - Optional username = Optional.empty(); - Optional password = Optional.empty(); - if (pluginConfig.hasPath(EsClusterConnectionConfig.USERNAME.key())) { - username = - Optional.of(pluginConfig.getString(EsClusterConnectionConfig.USERNAME.key())); - if (pluginConfig.hasPath(EsClusterConnectionConfig.PASSWORD.key())) { - password = - Optional.of( - pluginConfig.getString(EsClusterConnectionConfig.PASSWORD.key())); - } - } - - return createInstance(hosts, username, password); - } - - public static EsRestClient createInstance( - List hosts, Optional username, Optional password) { - RestClientBuilder restClientBuilder = getRestClientBuilder(hosts, username, password); - return new EsRestClient(restClientBuilder.build()); - } - - private static RestClientBuilder getRestClientBuilder( - List hosts, Optional username, Optional password) { - // HttpHost[] httpHosts = new HttpHost[hosts.size()]; - // for (int i = 0; i < hosts.size(); i++) { - // httpHosts[i] = HttpHost.create(hosts.get(i)); - // } - - String config = - "D:\\code\\seatunnel-dev\\incubator-seatunnel\\seatunnel-connectors-v2\\connector-elasticsearch-hw\\src\\main\\resources\\conf\\"; - return new HwRestClient(config).getRestClientBuilder(); - } - - public BulkResponse bulk(String requestBody) { - Request request = new Request("POST", "/_bulk"); - request.setJsonEntity(requestBody); - try { - Response response = restClient.performRequest(request); - if (response == null) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.BULK_RESPONSE_ERROR, - "bulk es Response is null"); - } - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - ObjectMapper objectMapper = new ObjectMapper(); - String entity = EntityUtils.toString(response.getEntity()); - JsonNode json = objectMapper.readTree(entity); - int took = json.get("took").asInt(); - boolean errors = json.get("errors").asBoolean(); - return new BulkResponse(errors, took, entity); - } else { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.BULK_RESPONSE_ERROR, - String.format( - "bulk es response status code=%d,request boy=%s", - response.getStatusLine().getStatusCode(), requestBody)); - } - } catch (IOException e) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.BULK_RESPONSE_ERROR, - String.format("bulk es error,request boy=%s", requestBody), - e); - } - } - - public ElasticsearchClusterInfo getClusterInfo() { - Request request = new Request("GET", "/"); - try { - Response response = restClient.performRequest(request); - String result = EntityUtils.toString(response.getEntity()); - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode jsonNode = objectMapper.readTree(result); - JsonNode versionNode = jsonNode.get("version"); - return ElasticsearchClusterInfo.builder() - .clusterVersion(versionNode.get("number").asText()) - .distribution( - Optional.ofNullable(versionNode.get("distribution")) - .map(e -> e.asText()) - .orElse(null)) - .build(); - } catch (IOException e) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.GET_ES_VERSION_FAILED, - "fail to get elasticsearch version.", - e); - } - } - - public void close() { - try { - restClient.close(); - } catch (IOException e) { - log.warn("close elasticsearch connection error", e); - } - } - - /** - * first time to request search documents by scroll call /${index}/_search?scroll=${scroll} - * - * @param index index name - * @param source select fields - * @param scrollTime such as:1m - * @param scrollSize fetch documents count in one request - */ - public ScrollResult searchByScroll( - String index, - List source, - Map query, - String scrollTime, - int scrollSize) { - Map param = new HashMap<>(); - param.put("query", query); - param.put("_source", source); - param.put("sort", new String[] {"_doc"}); - param.put("size", scrollSize); - String endpoint = "/" + index + "/_search?scroll=" + scrollTime; - ScrollResult scrollResult = - getDocsFromScrollRequest(endpoint, JsonUtils.toJsonString(param)); - return scrollResult; - } - - /** - * scroll to get result call _search/scroll - * - * @param scrollId the scroll id of the last request - * @param scrollTime such as:1m - */ - public ScrollResult searchWithScrollId(String scrollId, String scrollTime) { - Map param = new HashMap<>(); - param.put("scroll_id", scrollId); - param.put("scroll", scrollTime); - ScrollResult scrollResult = - getDocsFromScrollRequest("/_search/scroll", JsonUtils.toJsonString(param)); - return scrollResult; - } - - private ScrollResult getDocsFromScrollRequest(String endpoint, String requestBody) { - Request request = new Request("POST", endpoint); - request.setJsonEntity(requestBody); - try { - Response response = restClient.performRequest(request); - if (response == null) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.SCROLL_REQUEST_ERROR, - "POST " + endpoint + " response null"); - } - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - String entity = EntityUtils.toString(response.getEntity()); - ObjectNode responseJson = JsonUtils.parseObject(entity); - - JsonNode shards = responseJson.get("_shards"); - int totalShards = shards.get("total").intValue(); - int successful = shards.get("successful").intValue(); - Asserts.check( - totalShards == successful, - String.format( - "POST %s,total shards(%d)!= successful shards(%d)", - endpoint, totalShards, successful)); - - ScrollResult scrollResult = getDocsFromScrollResponse(responseJson); - return scrollResult; - } else { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.SCROLL_REQUEST_ERROR, - String.format( - "POST %s response status code=%d,request boy=%s", - endpoint, response.getStatusLine().getStatusCode(), requestBody)); - } - } catch (IOException e) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.SCROLL_REQUEST_ERROR, - String.format("POST %s error,request boy=%s", endpoint, requestBody), - e); - } - } - - private ScrollResult getDocsFromScrollResponse(ObjectNode responseJson) { - ScrollResult scrollResult = new ScrollResult(); - String scrollId = responseJson.get("_scroll_id").asText(); - scrollResult.setScrollId(scrollId); - - JsonNode hitsNode = responseJson.get("hits").get("hits"); - List> docs = new ArrayList<>(hitsNode.size()); - scrollResult.setDocs(docs); - - Iterator iter = hitsNode.iterator(); - while (iter.hasNext()) { - Map doc = new HashMap<>(); - JsonNode hitNode = iter.next(); - doc.put("_index", hitNode.get("_index").textValue()); - doc.put("_id", hitNode.get("_id").textValue()); - JsonNode source = hitNode.get("_source"); - for (Iterator> iterator = source.fields(); - iterator.hasNext(); ) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - if (entry.getValue() instanceof TextNode) { - doc.put(fieldName, entry.getValue().textValue()); - } else { - doc.put(fieldName, entry.getValue()); - } - } - docs.add(doc); - } - return scrollResult; - } - - public List getIndexDocsCount(String index) { - String endpoint = String.format("/_cat/indices/%s?h=index,docsCount&format=json", index); - Request request = new Request("GET", endpoint); - try { - Response response = restClient.performRequest(request); - if (response == null) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.GET_INDEX_DOCS_COUNT_FAILED, - "GET " + endpoint + " response null"); - } - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - String entity = EntityUtils.toString(response.getEntity()); - List indexDocsCounts = - JsonUtils.toList(entity, IndexDocsCount.class); - return indexDocsCounts; - } else { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.GET_INDEX_DOCS_COUNT_FAILED, - String.format( - "GET %s response status code=%d", - endpoint, response.getStatusLine().getStatusCode())); - } - } catch (IOException ex) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.GET_INDEX_DOCS_COUNT_FAILED, ex); - } - } - - public List listIndex() { - String endpoint = "/_cat/indices?format=json"; - Request request = new Request("GET", endpoint); - try { - Response response = restClient.performRequest(request); - if (response == null) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.LIST_INDEX_FAILED, - "GET " + endpoint + " response null"); - } - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - String entity = EntityUtils.toString(response.getEntity()); - return JsonUtils.toList(entity, Map.class).stream() - .map(map -> map.get("index").toString()) - .collect(Collectors.toList()); - } else { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.LIST_INDEX_FAILED, - String.format( - "GET %s response status code=%d", - endpoint, response.getStatusLine().getStatusCode())); - } - } catch (IOException ex) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.LIST_INDEX_FAILED, ex); - } - } - - // todo: We don't support set the index mapping now. - public void createIndex(String indexName) { - String endpoint = String.format("/%s", indexName); - Request request = new Request("PUT", endpoint); - try { - Response response = restClient.performRequest(request); - if (response == null) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.CREATE_INDEX_FAILED, - "PUT " + endpoint + " response null"); - } - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.CREATE_INDEX_FAILED, - String.format( - "PUT %s response status code=%d", - endpoint, response.getStatusLine().getStatusCode())); - } - } catch (IOException ex) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.CREATE_INDEX_FAILED, ex); - } - } - - public void dropIndex(String tableName) { - String endpoint = String.format("/%s", tableName); - Request request = new Request("DELETE", endpoint); - try { - Response response = restClient.performRequest(request); - if (response == null) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.DROP_INDEX_FAILED, - "DELETE " + endpoint + " response null"); - } - // todo: if the index doesn't exist, the response status code is 200? - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - return; - } else { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.DROP_INDEX_FAILED, - String.format( - "DELETE %s response status code=%d", - endpoint, response.getStatusLine().getStatusCode())); - } - } catch (IOException ex) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.DROP_INDEX_FAILED, ex); - } - } - - /** - * get es field name and type mapping realtion - * - * @param index index name - * @return {key-> field name,value->es type} - */ - public Map getFieldTypeMapping(String index, List source) { - String endpoint = String.format("/%s/_mappings", index); - Request request = new Request("GET", endpoint); - Map mapping = new HashMap<>(); - try { - Response response = restClient.performRequest(request); - if (response == null) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.GET_INDEX_DOCS_COUNT_FAILED, - "GET " + endpoint + " response null"); - } - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.GET_INDEX_DOCS_COUNT_FAILED, - String.format( - "GET %s response status code=%d", - endpoint, response.getStatusLine().getStatusCode())); - } - String entity = EntityUtils.toString(response.getEntity()); - log.info(String.format("GET %s respnse=%s", endpoint, entity)); - ObjectNode responseJson = JsonUtils.parseObject(entity); - for (Iterator it = responseJson.elements(); it.hasNext(); ) { - JsonNode indexProperty = it.next(); - JsonNode mappingsProperty = indexProperty.get("mappings"); - if (mappingsProperty.has("mappingsProperty")) { - JsonNode properties = mappingsProperty.get("properties"); - mapping = getFieldTypeMappingFromProperties(properties, source); - } else { - for (JsonNode typeNode : mappingsProperty) { - JsonNode properties; - if (typeNode.has("properties")) { - properties = typeNode.get("properties"); - } else { - properties = typeNode; - } - mapping.putAll(getFieldTypeMappingFromProperties(properties, source)); - } - } - } - } catch (IOException ex) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.GET_INDEX_DOCS_COUNT_FAILED, ex); - } - return mapping; - } - - private static Map getFieldTypeMappingFromProperties( - JsonNode properties, List source) { - Map allElasticSearchFieldTypeInfoMap = new HashMap<>(); - properties - .fields() - .forEachRemaining( - entry -> { - String fieldName = entry.getKey(); - JsonNode fieldProperty = entry.getValue(); - if (fieldProperty.has("type")) { - allElasticSearchFieldTypeInfoMap.put( - fieldName, fieldProperty.get("type").asText()); - } - }); - if (CollectionUtils.isEmpty(source)) { - return allElasticSearchFieldTypeInfoMap; - } - - return source.stream() - .collect( - Collectors.toMap( - Function.identity(), - fieldName -> { - String fieldType = - allElasticSearchFieldTypeInfoMap.get(fieldName); - if (fieldType == null) { - log.warn( - "fail to get elasticsearch field {} mapping type,so give a default type text", - fieldName); - return "text"; - } - return fieldType; - })); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/EsClusterConnectionConfig.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/EsClusterConnectionConfig.java deleted file mode 100644 index eb16c28173a..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/EsClusterConnectionConfig.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.Options; - -import java.util.List; - -public class EsClusterConnectionConfig { - - public static final Option> HOSTS = - Options.key("hosts") - .listType() - .noDefaultValue() - .withDescription( - "Elasticsearch cluster http address, the format is host:port, allowing multiple hosts to be specified. Such as [\"host1:9200\", \"host2:9200\"]"); - - public static final Option USERNAME = - Options.key("username") - .stringType() - .noDefaultValue() - .withDescription("x-pack username"); - - public static final Option PASSWORD = - Options.key("password") - .stringType() - .noDefaultValue() - .withDescription("x-pack password"); - - public static final Option TLS_VERIFY_CERTIFICATE = - Options.key("tls_verify_certificate") - .booleanType() - .defaultValue(true) - .withDescription("Enable certificates validation for HTTPS endpoints"); - - public static final Option TLS_VERIFY_HOSTNAME = - Options.key("tls_verify_hostname") - .booleanType() - .defaultValue(true) - .withDescription("Enable hostname validation for HTTPS endpoints"); - - public static final Option TLS_KEY_STORE_PATH = - Options.key("tls_keystore_path") - .stringType() - .noDefaultValue() - .withDescription( - "The path to the PEM or JKS key store. This file must be readable by the operating system user running SeaTunnel."); - - public static final Option TLS_KEY_STORE_PASSWORD = - Options.key("tls_keystore_password") - .stringType() - .noDefaultValue() - .withDescription("The key password for the key store specified"); - - public static final Option TLS_TRUST_STORE_PATH = - Options.key("tls_truststore_path") - .stringType() - .noDefaultValue() - .withDescription( - "The path to PEM or JKS trust store. This file must be readable by the operating system user running SeaTunnel."); - - public static final Option TLS_TRUST_STORE_PASSWORD = - Options.key("tls_truststore_password") - .stringType() - .noDefaultValue() - .withDescription("The key password for the trust store specified"); -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/SinkConfig.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/SinkConfig.java deleted file mode 100644 index c507b86cd4e..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/SinkConfig.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.Options; - -import java.util.List; - -public class SinkConfig { - - public static final Option INDEX = - Options.key("index") - .stringType() - .noDefaultValue() - .withDescription( - "Elasticsearch index name.Index support contains variables of field name,such as seatunnel_${age},and the field must appear at seatunnel row. If not, we will treat it as a normal index"); - - public static final Option INDEX_TYPE = - Options.key("index_type") - .stringType() - .noDefaultValue() - .withDescription( - "Elasticsearch index type, it is recommended not to specify in elasticsearch 6 and above"); - - public static final Option> PRIMARY_KEYS = - Options.key("primary_keys") - .listType(String.class) - .noDefaultValue() - .withDescription("Primary key fields used to generate the document `_id`"); - - public static final Option KEY_DELIMITER = - Options.key("key_delimiter") - .stringType() - .defaultValue("_") - .withDescription( - "Delimiter for composite keys (\"_\" by default), e.g., \"$\" would result in document `_id` \"KEY1$KEY2$KEY3\"."); - - public static final Option MAX_BATCH_SIZE = - Options.key("max_batch_size") - .intType() - .defaultValue(10) - .withDescription("batch bulk doc max size"); - - public static final Option MAX_RETRY_COUNT = - Options.key("max_retry_count") - .intType() - .defaultValue(3) - .withDescription("one bulk request max try count"); -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/SourceConfig.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/SourceConfig.java deleted file mode 100644 index 6e4fdf7d088..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/config/SourceConfig.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.Options; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class SourceConfig { - - public static final Option INDEX = - Options.key("index") - .stringType() - .noDefaultValue() - .withDescription("Elasticsearch index name, support * fuzzy matching"); - - public static final Option> SOURCE = - Options.key("source") - .listType() - .noDefaultValue() - .withDescription( - "The fields of index. You can get the document id by specifying the field _id.If sink _id to other index,you need specify an alias for _id due to the Elasticsearch limit"); - - public static final Option SCROLL_TIME = - Options.key("scroll_time") - .stringType() - .defaultValue("1m") - .withDescription( - "Amount of time Elasticsearch will keep the search context alive for scroll requests"); - - public static final Option SCROLL_SIZE = - Options.key("scroll_size") - .intType() - .defaultValue(100) - .withDescription( - "Maximum number of hits to be returned with each Elasticsearch scroll request"); - - public static final Option QUERY = - Options.key("query") - .objectType(Map.class) - .defaultValue( - Collections.singletonMap("match_all", new HashMap())) - .withDescription( - "Elasticsearch query language. You can control the range of data read"); -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/constant/ElasticsearchVersion.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/constant/ElasticsearchVersion.java deleted file mode 100644 index c4ca7272cc4..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/constant/ElasticsearchVersion.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.constant; - -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorException; - -public enum ElasticsearchVersion { - ES2(2), - ES5(5), - ES6(6), - ES7(7), - ES8(8); - - private int version; - - ElasticsearchVersion(int version) { - this.version = version; - } - - public int getVersion() { - return version; - } - - public void setVersion(int version) { - this.version = version; - } - - public static ElasticsearchVersion get(int version) { - for (ElasticsearchVersion elasticsearchVersion : ElasticsearchVersion.values()) { - if (elasticsearchVersion.getVersion() == version) { - return elasticsearchVersion; - } - } - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.GET_ES_VERSION_FAILED, - String.format("version=%d,fail fo find ElasticsearchVersion.", version)); - } - - public static ElasticsearchVersion get(String clusterVersion) { - String[] versionArr = clusterVersion.split("\\."); - int version = Integer.parseInt(versionArr[0]); - return get(version); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/constant/EsTypeMappingSeaTunnelType.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/constant/EsTypeMappingSeaTunnelType.java deleted file mode 100644 index d525f03b3d4..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/constant/EsTypeMappingSeaTunnelType.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.constant; - -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.LocalTimeType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorException; - -import java.util.HashMap; -import java.util.Map; - -public class EsTypeMappingSeaTunnelType { - - private static final Map MAPPING = - new HashMap() { - { - put("string", BasicType.STRING_TYPE); - put("keyword", BasicType.STRING_TYPE); - put("text", BasicType.STRING_TYPE); - put("binary", BasicType.STRING_TYPE); - put("boolean", BasicType.BOOLEAN_TYPE); - put("byte", BasicType.BYTE_TYPE); - put("short", BasicType.SHORT_TYPE); - put("integer", BasicType.INT_TYPE); - put("long", BasicType.LONG_TYPE); - put("float", BasicType.FLOAT_TYPE); - put("half_float", BasicType.FLOAT_TYPE); - put("double", BasicType.DOUBLE_TYPE); - put("date", LocalTimeType.LOCAL_DATE_TIME_TYPE); - } - }; - - /** - * if not find the mapping SeaTunnelDataType will throw runtime exception - * - * @param esType - * @return - */ - public static SeaTunnelDataType getSeaTunnelDataType(String esType) { - SeaTunnelDataType seaTunnelDataType = MAPPING.get(esType); - if (seaTunnelDataType == null) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.ES_FIELD_TYPE_NOT_SUPPORT, - String.format("elasticsearch type is %s", esType)); - } - return seaTunnelDataType; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/BulkResponse.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/BulkResponse.java deleted file mode 100644 index 1ff65eb4484..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/BulkResponse.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto; - -/** the response of bulk ES by http request */ -public class BulkResponse { - - private boolean errors; - private int took; - private String response; - - public BulkResponse() {} - - public BulkResponse(boolean errors, int took, String response) { - this.errors = errors; - this.took = took; - this.response = response; - } - - public boolean isErrors() { - return errors; - } - - public void setErrors(boolean errors) { - this.errors = errors; - } - - public int getTook() { - return took; - } - - public void setTook(int took) { - this.took = took; - } - - public String getResponse() { - return response; - } - - public void setResponse(String response) { - this.response = response; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/ElasticsearchClusterInfo.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/ElasticsearchClusterInfo.java deleted file mode 100644 index 09f703c6a9b..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/ElasticsearchClusterInfo.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto; - -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.constant.ElasticsearchVersion; - -import com.google.common.base.Strings; -import lombok.Builder; -import lombok.Getter; -import lombok.ToString; - -@Getter -@Builder -@ToString -public class ElasticsearchClusterInfo { - private String distribution; - private String clusterVersion; - - public ElasticsearchVersion getElasticsearchVersion() { - return ElasticsearchVersion.get(clusterVersion); - } - - public boolean isOpensearch() { - return !Strings.isNullOrEmpty(distribution) && "opensearch".equalsIgnoreCase(distribution); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/IndexInfo.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/IndexInfo.java deleted file mode 100644 index 194157d7efd..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/IndexInfo.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.SinkConfig; - -import lombok.Data; - -/** index config by seatunnel */ -@Data -public class IndexInfo { - - private String index; - private String type; - private String[] primaryKeys; - private String keyDelimiter; - - public IndexInfo(Config pluginConfig) { - index = pluginConfig.getString(SinkConfig.INDEX.key()); - if (pluginConfig.hasPath(SinkConfig.INDEX_TYPE.key())) { - type = pluginConfig.getString(SinkConfig.INDEX_TYPE.key()); - } - if (pluginConfig.hasPath(SinkConfig.PRIMARY_KEYS.key())) { - primaryKeys = - pluginConfig - .getStringList(SinkConfig.PRIMARY_KEYS.key()) - .toArray(new String[0]); - } - keyDelimiter = SinkConfig.KEY_DELIMITER.defaultValue(); - if (pluginConfig.hasPath(SinkConfig.KEY_DELIMITER.key())) { - keyDelimiter = pluginConfig.getString(SinkConfig.KEY_DELIMITER.key()); - } - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/IndexDocsCount.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/IndexDocsCount.java deleted file mode 100644 index 585b5638015..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/IndexDocsCount.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source; - -public class IndexDocsCount { - - private String index; - /** index docs count */ - private Long docsCount; - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public Long getDocsCount() { - return docsCount; - } - - public void setDocsCount(Long docsCount) { - this.docsCount = docsCount; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/ScrollResult.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/ScrollResult.java deleted file mode 100644 index c93f811b61e..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/ScrollResult.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source; - -import lombok.Data; - -import java.util.List; -import java.util.Map; - -@Data -public class ScrollResult { - - private String scrollId; - private List> docs; -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/SourceIndexInfo.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/SourceIndexInfo.java deleted file mode 100644 index 3559b57ed45..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/dto/source/SourceIndexInfo.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -@Data -@AllArgsConstructor -public class SourceIndexInfo implements Serializable { - private String index; - private List source; - private Map query; - private String scrollTime; - private int scrollSize; -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/exception/ElasticsearchConnectorErrorCode.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/exception/ElasticsearchConnectorErrorCode.java deleted file mode 100644 index d2f382a6bb3..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/exception/ElasticsearchConnectorErrorCode.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception; - -import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; - -public enum ElasticsearchConnectorErrorCode implements SeaTunnelErrorCode { - BULK_RESPONSE_ERROR("ELASTICSEARCH-01", "Bulk es response error"), - GET_ES_VERSION_FAILED("ELASTICSEARCH-02", "Get elasticsearch version failed"), - SCROLL_REQUEST_ERROR("ELASTICSEARCH-03", "Fail to scroll request"), - GET_INDEX_DOCS_COUNT_FAILED( - "ELASTICSEARCH-04", "Get elasticsearch document index count failed"), - LIST_INDEX_FAILED("ELASTICSEARCH-05", "List elasticsearch index failed"), - DROP_INDEX_FAILED("ELASTICSEARCH-06", "Drop elasticsearch index failed"), - CREATE_INDEX_FAILED("ELASTICSEARCH-07", "Create elasticsearch index failed"), - ES_FIELD_TYPE_NOT_SUPPORT("ELASTICSEARCH-08", "Not support the elasticsearch field type"); - ; - - private final String code; - private final String description; - - ElasticsearchConnectorErrorCode(String code, String description) { - this.code = code; - this.description = description; - } - - @Override - public String getCode() { - return code; - } - - @Override - public String getDescription() { - return description; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/exception/ElasticsearchConnectorException.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/exception/ElasticsearchConnectorException.java deleted file mode 100644 index d4bffb7d882..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/exception/ElasticsearchConnectorException.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception; - -import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; -import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; - -public class ElasticsearchConnectorException extends SeaTunnelRuntimeException { - public ElasticsearchConnectorException( - SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage) { - super(seaTunnelErrorCode, errorMessage); - } - - public ElasticsearchConnectorException( - SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage, Throwable cause) { - super(seaTunnelErrorCode, errorMessage, cause); - } - - public ElasticsearchConnectorException(SeaTunnelErrorCode seaTunnelErrorCode, Throwable cause) { - super(seaTunnelErrorCode, cause); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/ElasticsearchRowSerializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/ElasticsearchRowSerializer.java deleted file mode 100644 index ae9de359f33..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/ElasticsearchRowSerializer.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize; - -import org.apache.seatunnel.shade.com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; - -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.ElasticsearchClusterInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.IndexInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorException; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index.IndexSerializer; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index.IndexSerializerFactory; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type.IndexTypeSerializer; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type.IndexTypeSerializerFactory; - -import lombok.NonNull; - -import java.time.temporal.Temporal; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -/** use in elasticsearch version >= 2.x and <= 8.x */ -public class ElasticsearchRowSerializer implements SeaTunnelRowSerializer { - private final SeaTunnelRowType seaTunnelRowType; - private final ObjectMapper objectMapper = new ObjectMapper(); - - private final IndexSerializer indexSerializer; - - private final IndexTypeSerializer indexTypeSerializer; - private final Function keyExtractor; - - public ElasticsearchRowSerializer( - ElasticsearchClusterInfo elasticsearchClusterInfo, - IndexInfo indexInfo, - SeaTunnelRowType seaTunnelRowType) { - this.indexTypeSerializer = - IndexTypeSerializerFactory.getIndexTypeSerializer( - elasticsearchClusterInfo, indexInfo.getType()); - this.indexSerializer = - IndexSerializerFactory.getIndexSerializer(indexInfo.getIndex(), seaTunnelRowType); - this.seaTunnelRowType = seaTunnelRowType; - this.keyExtractor = - KeyExtractor.createKeyExtractor( - seaTunnelRowType, indexInfo.getPrimaryKeys(), indexInfo.getKeyDelimiter()); - } - - @Override - public String serializeRow(SeaTunnelRow row) { - switch (row.getRowKind()) { - case INSERT: - case UPDATE_AFTER: - return serializeUpsert(row); - case UPDATE_BEFORE: - case DELETE: - return serializeDelete(row); - default: - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION, - "Unsupported write row kind: " + row.getRowKind()); - } - } - - private String serializeUpsert(SeaTunnelRow row) { - String key = keyExtractor.apply(row); - Map document = toDocumentMap(row); - - try { - if (key != null) { - Map upsertMetadata = createMetadata(row, key); - /** - * format example: { "update" : {"_index" : "${your_index}", "_id" : - * "${your_document_id}"} }\n { "doc" : ${your_document_json}, "doc_as_upsert" : - * true } - */ - return new StringBuilder() - .append("{ \"update\" :") - .append(objectMapper.writeValueAsString(upsertMetadata)) - .append("}") - .append("\n") - .append("{ \"doc\" :") - .append(objectMapper.writeValueAsString(document)) - .append(", \"doc_as_upsert\" : true }") - .toString(); - } else { - Map indexMetadata = createMetadata(row); - /** - * format example: { "index" : {"_index" : "${your_index}", "_id" : - * "${your_document_id}"} }\n ${your_document_json} - */ - return new StringBuilder() - .append("{ \"index\" :") - .append(objectMapper.writeValueAsString(indexMetadata)) - .append("}") - .append("\n") - .append(objectMapper.writeValueAsString(document)) - .toString(); - } - } catch (JsonProcessingException e) { - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.JSON_OPERATION_FAILED, - "Object json deserialization exception.", - e); - } - } - - private String serializeDelete(SeaTunnelRow row) { - String key = keyExtractor.apply(row); - Map deleteMetadata = createMetadata(row, key); - try { - /** - * format example: { "delete" : {"_index" : "${your_index}", "_id" : - * "${your_document_id}"} } - */ - return new StringBuilder() - .append("{ \"delete\" :") - .append(objectMapper.writeValueAsString(deleteMetadata)) - .append("}") - .toString(); - } catch (JsonProcessingException e) { - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.JSON_OPERATION_FAILED, - "Object json deserialization exception.", - e); - } - } - - private Map toDocumentMap(SeaTunnelRow row) { - String[] fieldNames = seaTunnelRowType.getFieldNames(); - Map doc = new HashMap<>(fieldNames.length); - Object[] fields = row.getFields(); - for (int i = 0; i < fieldNames.length; i++) { - Object value = fields[i]; - if (value instanceof Temporal) { - // jackson not support jdk8 new time api - doc.put(fieldNames[i], value.toString()); - } else { - doc.put(fieldNames[i], value); - } - } - return doc; - } - - private Map createMetadata(@NonNull SeaTunnelRow row, @NonNull String key) { - Map actionMetadata = createMetadata(row); - actionMetadata.put("_id", key); - return actionMetadata; - } - - private Map createMetadata(@NonNull SeaTunnelRow row) { - Map actionMetadata = new HashMap<>(2); - actionMetadata.put("_index", indexSerializer.serialize(row)); - indexTypeSerializer.fillType(actionMetadata); - return actionMetadata; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/KeyExtractor.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/KeyExtractor.java deleted file mode 100644 index 99612069da0..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/KeyExtractor.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize; - -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorException; - -import lombok.AllArgsConstructor; - -import java.io.Serializable; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; - -@AllArgsConstructor -public class KeyExtractor implements Function, Serializable { - private final FieldFormatter[] fieldFormatters; - private final String keyDelimiter; - - @Override - public String apply(SeaTunnelRow row) { - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < fieldFormatters.length; i++) { - if (i > 0) { - builder.append(keyDelimiter); - } - String value = fieldFormatters[i].format(row); - builder.append(value); - } - return builder.toString(); - } - - public static Function createKeyExtractor( - SeaTunnelRowType rowType, String[] primaryKeys, String keyDelimiter) { - if (primaryKeys == null) { - return row -> null; - } - - List fieldFormatters = new ArrayList<>(primaryKeys.length); - for (String fieldName : primaryKeys) { - int fieldIndex = rowType.indexOf(fieldName); - SeaTunnelDataType fieldType = rowType.getFieldType(fieldIndex); - FieldFormatter fieldFormatter = createFieldFormatter(fieldIndex, fieldType); - fieldFormatters.add(fieldFormatter); - } - return new KeyExtractor(fieldFormatters.toArray(new FieldFormatter[0]), keyDelimiter); - } - - private static FieldFormatter createFieldFormatter( - int fieldIndex, SeaTunnelDataType fieldType) { - return row -> { - switch (fieldType.getSqlType()) { - case ROW: - case ARRAY: - case MAP: - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION, - "Unsupported type: " + fieldType); - case DATE: - LocalDate localDate = (LocalDate) row.getField(fieldIndex); - return localDate.toString(); - case TIME: - LocalTime localTime = (LocalTime) row.getField(fieldIndex); - return localTime.toString(); - case TIMESTAMP: - LocalDateTime localDateTime = (LocalDateTime) row.getField(fieldIndex); - return localDateTime.toString(); - default: - return row.getField(fieldIndex).toString(); - } - }; - } - - private interface FieldFormatter extends Serializable { - String format(SeaTunnelRow row); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/SeaTunnelRowSerializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/SeaTunnelRowSerializer.java deleted file mode 100644 index 695676b4788..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/SeaTunnelRowSerializer.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize; - -import org.apache.seatunnel.api.table.type.SeaTunnelRow; - -public interface SeaTunnelRowSerializer { - - String serializeRow(SeaTunnelRow row); -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/IndexSerializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/IndexSerializer.java deleted file mode 100644 index ad1637022d9..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/IndexSerializer.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index; - -import org.apache.seatunnel.api.table.type.SeaTunnelRow; - -/** index is a variable */ -public interface IndexSerializer { - - String serialize(SeaTunnelRow row); -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/IndexSerializerFactory.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/IndexSerializerFactory.java deleted file mode 100644 index d187833cdc5..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/IndexSerializerFactory.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index; - -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index.impl.FixedValueIndexSerializer; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index.impl.VariableIndexSerializer; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.util.RegexUtils; - -import java.util.List; - -public class IndexSerializerFactory { - - public static IndexSerializer getIndexSerializer( - String index, SeaTunnelRowType seaTunnelRowType) { - List fieldNames = RegexUtils.extractDatas(index, "\\$\\{(.*?)\\}"); - if (fieldNames != null && fieldNames.size() > 0) { - return new VariableIndexSerializer(seaTunnelRowType, index, fieldNames); - } else { - return new FixedValueIndexSerializer(index); - } - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/impl/FixedValueIndexSerializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/impl/FixedValueIndexSerializer.java deleted file mode 100644 index 4cdb92da9d9..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/impl/FixedValueIndexSerializer.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index.impl; - -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index.IndexSerializer; - -/** index is a fixed value,not a variable */ -public class FixedValueIndexSerializer implements IndexSerializer { - - private final String index; - - public FixedValueIndexSerializer(String index) { - this.index = index; - } - - @Override - public String serialize(SeaTunnelRow row) { - return index; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/impl/VariableIndexSerializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/impl/VariableIndexSerializer.java deleted file mode 100644 index 62a89483afe..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/index/impl/VariableIndexSerializer.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index.impl; - -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.index.IndexSerializer; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** index include variable */ -public class VariableIndexSerializer implements IndexSerializer { - - private final String index; - private final Map fieldIndexMap; - - private final String nullDefault = "null"; - - public VariableIndexSerializer( - SeaTunnelRowType seaTunnelRowType, String index, List fieldNames) { - this.index = index; - String[] rowFieldNames = seaTunnelRowType.getFieldNames(); - fieldIndexMap = new HashMap<>(rowFieldNames.length); - for (int i = 0; i < rowFieldNames.length; i++) { - if (fieldNames.contains(rowFieldNames[i])) { - fieldIndexMap.put(rowFieldNames[i], i); - } - } - } - - @Override - public String serialize(SeaTunnelRow row) { - String indexName = this.index; - for (Map.Entry fieldIndexEntry : fieldIndexMap.entrySet()) { - String fieldName = fieldIndexEntry.getKey(); - int fieldIndex = fieldIndexEntry.getValue(); - String value = getValue(fieldIndex, row); - indexName = indexName.replace(String.format("${%s}", fieldName), value); - } - return indexName.toLowerCase(); - } - - private String getValue(int fieldIndex, SeaTunnelRow row) { - Object valueObj = row.getField(fieldIndex); - if (valueObj == null) { - return nullDefault; - } else { - return valueObj.toString(); - } - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/DefaultSeaTunnelRowDeserializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/DefaultSeaTunnelRowDeserializer.java deleted file mode 100644 index c8899de9e49..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/DefaultSeaTunnelRowDeserializer.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.source; - -import org.apache.seatunnel.shade.com.fasterxml.jackson.core.JsonProcessingException; -import org.apache.seatunnel.shade.com.fasterxml.jackson.core.type.TypeReference; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.node.TextNode; - -import org.apache.seatunnel.api.table.type.ArrayType; -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.DecimalType; -import org.apache.seatunnel.api.table.type.LocalTimeType; -import org.apache.seatunnel.api.table.type.MapType; -import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; -import org.apache.seatunnel.common.utils.JsonUtils; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorException; - -import java.lang.reflect.Array; -import java.math.BigDecimal; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Base64; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.seatunnel.api.table.type.BasicType.BOOLEAN_TYPE; -import static org.apache.seatunnel.api.table.type.BasicType.BYTE_TYPE; -import static org.apache.seatunnel.api.table.type.BasicType.DOUBLE_TYPE; -import static org.apache.seatunnel.api.table.type.BasicType.FLOAT_TYPE; -import static org.apache.seatunnel.api.table.type.BasicType.INT_TYPE; -import static org.apache.seatunnel.api.table.type.BasicType.LONG_TYPE; -import static org.apache.seatunnel.api.table.type.BasicType.SHORT_TYPE; -import static org.apache.seatunnel.api.table.type.BasicType.STRING_TYPE; -import static org.apache.seatunnel.api.table.type.BasicType.VOID_TYPE; - -public class DefaultSeaTunnelRowDeserializer implements SeaTunnelRowDeserializer { - - private final SeaTunnelRowType rowTypeInfo; - - private final ObjectMapper mapper = new ObjectMapper(); - - private final Map dateTimeFormatterMap = - new HashMap() { - { - put("yyyy-MM-dd HH".length(), DateTimeFormatter.ofPattern("yyyy-MM-dd HH")); - put( - "yyyy-MM-dd HH:mm".length(), - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")); - put( - "yyyyMMdd HH:mm:ss".length(), - DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss")); - put( - "yyyy-MM-dd HH:mm:ss".length(), - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); - put( - "yyyy-MM-dd HH:mm:ss.S".length(), - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.S")); - put( - "yyyy-MM-dd HH:mm:ss.SS".length(), - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SS")); - put( - "yyyy-MM-dd HH:mm:ss.SSS".length(), - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")); - put( - "yyyy-MM-dd HH:mm:ss.SSSS".length(), - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSS")); - put( - "yyyy-MM-dd HH:mm:ss.SSSSSS".length(), - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS")); - } - }; - - public DefaultSeaTunnelRowDeserializer(SeaTunnelRowType rowTypeInfo) { - this.rowTypeInfo = rowTypeInfo; - } - - @Override - public SeaTunnelRow deserialize(ElasticsearchRecord rowRecord) { - return convert(rowRecord); - } - - SeaTunnelRow convert(ElasticsearchRecord rowRecord) { - Object[] seaTunnelFields = new Object[rowTypeInfo.getTotalFields()]; - String fieldName = null; - Object value = null; - SeaTunnelDataType seaTunnelDataType = null; - try { - for (int i = 0; i < rowTypeInfo.getTotalFields(); i++) { - fieldName = rowTypeInfo.getFieldName(i); - value = recursiveGet(rowRecord.getDoc(), fieldName); - if (value != null) { - seaTunnelDataType = rowTypeInfo.getFieldType(i); - if (value instanceof TextNode) { - seaTunnelFields[i] = - convertValue(seaTunnelDataType, ((TextNode) value).textValue()); - } else { - seaTunnelFields[i] = convertValue(seaTunnelDataType, value.toString()); - } - } - } - } catch (Exception ex) { - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION, - String.format( - "error fieldName=%s,fieldValue=%s,seaTunnelDataType=%s,rowRecord=%s", - fieldName, value, seaTunnelDataType, JsonUtils.toJsonString(rowRecord)), - ex); - } - return new SeaTunnelRow(seaTunnelFields); - } - - Object convertValue(SeaTunnelDataType fieldType, String fieldValue) - throws JsonProcessingException { - if (BOOLEAN_TYPE.equals(fieldType)) { - return Boolean.parseBoolean(fieldValue); - } else if (BYTE_TYPE.equals(fieldType)) { - return Byte.valueOf(fieldValue); - } else if (SHORT_TYPE.equals(fieldType)) { - return Short.parseShort(fieldValue); - } else if (INT_TYPE.equals(fieldType)) { - return Integer.parseInt(fieldValue); - } else if (LONG_TYPE.equals(fieldType)) { - return Long.parseLong(fieldValue); - } else if (FLOAT_TYPE.equals(fieldType)) { - return Float.parseFloat(fieldValue); - } else if (DOUBLE_TYPE.equals(fieldType)) { - return Double.parseDouble(fieldValue); - } else if (STRING_TYPE.equals(fieldType)) { - return fieldValue; - } else if (LocalTimeType.LOCAL_DATE_TYPE.equals(fieldType)) { - LocalDateTime localDateTime = parseDate(fieldValue); - return localDateTime.toLocalDate(); - } else if (LocalTimeType.LOCAL_TIME_TYPE.equals(fieldType)) { - LocalDateTime localDateTime = parseDate(fieldValue); - return localDateTime.toLocalTime(); - } else if (LocalTimeType.LOCAL_DATE_TIME_TYPE.equals(fieldType)) { - return parseDate(fieldValue); - } else if (fieldType instanceof DecimalType) { - return new BigDecimal(fieldValue); - } else if (fieldType instanceof ArrayType) { - ArrayType arrayType = (ArrayType) fieldType; - BasicType elementType = arrayType.getElementType(); - List stringList = JsonUtils.toList(fieldValue, String.class); - Object arr = Array.newInstance(elementType.getTypeClass(), stringList.size()); - for (int i = 0; i < stringList.size(); i++) { - Object convertValue = convertValue(elementType, stringList.get(i)); - Array.set(arr, i, convertValue); - } - return arr; - } else if (fieldType instanceof MapType) { - MapType mapType = (MapType) fieldType; - SeaTunnelDataType keyType = mapType.getKeyType(); - - SeaTunnelDataType valueType = mapType.getValueType(); - Map stringMap = - mapper.readValue(fieldValue, new TypeReference>() {}); - Map convertMap = new HashMap(); - for (Map.Entry entry : stringMap.entrySet()) { - Object convertKey = convertValue(keyType, entry.getKey()); - Object convertValue = convertValue(valueType, entry.getValue()); - convertMap.put(convertKey, convertValue); - } - return convertMap; - } else if (fieldType instanceof PrimitiveByteArrayType) { - return Base64.getDecoder().decode(fieldValue); - } else if (VOID_TYPE.equals(fieldType) || fieldType == null) { - return null; - } else { - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.UNSUPPORTED_DATA_TYPE, - "Unexpected value: " + fieldType); - } - } - - private LocalDateTime parseDate(String fieldValue) { - // handle strings of timestamp type - try { - long ts = Long.parseLong(fieldValue); - return LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneId.systemDefault()); - } catch (NumberFormatException e) { - // no op - } - String formatDate = fieldValue.replace("T", " "); - if (fieldValue.length() == "yyyyMMdd".length() - || fieldValue.length() == "yyyy-MM-dd".length()) { - formatDate = fieldValue + " 00:00:00"; - } - DateTimeFormatter dateTimeFormatter = dateTimeFormatterMap.get(formatDate.length()); - if (dateTimeFormatter == null) { - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION, "unsupported date format"); - } - return LocalDateTime.parse(formatDate, dateTimeFormatter); - } - - Object recursiveGet(Map collect, String keyWithRecursive) { - Object value = null; - boolean isFirst = true; - for (String key : keyWithRecursive.split("\\.")) { - if (isFirst) { - value = collect.get(key); - isFirst = false; - } else if (value instanceof ObjectNode) { - value = ((ObjectNode) value).get(key); - } - } - return value; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/ElasticsearchRecord.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/ElasticsearchRecord.java deleted file mode 100644 index 6824031daf6..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/ElasticsearchRecord.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.source; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; - -import java.util.List; -import java.util.Map; - -@Getter -@ToString -@AllArgsConstructor -public class ElasticsearchRecord { - private Map doc; - private List source; -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/SeaTunnelRowDeserializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/SeaTunnelRowDeserializer.java deleted file mode 100644 index 03f22d2c60b..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/source/SeaTunnelRowDeserializer.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.source; - -import org.apache.seatunnel.api.table.type.SeaTunnelRow; - -public interface SeaTunnelRowDeserializer { - - SeaTunnelRow deserialize(ElasticsearchRecord rowRecord); -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/IndexTypeSerializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/IndexTypeSerializer.java deleted file mode 100644 index 22484f7b9a9..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/IndexTypeSerializer.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type; - -import java.util.Map; - -public interface IndexTypeSerializer { - - void fillType(Map indexInner); -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/IndexTypeSerializerFactory.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/IndexTypeSerializerFactory.java deleted file mode 100644 index f8cf55b9aa2..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/IndexTypeSerializerFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type; - -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.constant.ElasticsearchVersion; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.ElasticsearchClusterInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type.impl.NotIndexTypeSerializer; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type.impl.RequiredIndexTypeSerializer; - -public class IndexTypeSerializerFactory { - - private static final String DEFAULT_TYPE = "st"; - - private IndexTypeSerializerFactory() {} - - public static IndexTypeSerializer getIndexTypeSerializer( - ElasticsearchClusterInfo elasticsearchClusterInfo, String type) { - if (elasticsearchClusterInfo.isOpensearch()) { - return new NotIndexTypeSerializer(); - } - ElasticsearchVersion elasticsearchVersion = - elasticsearchClusterInfo.getElasticsearchVersion(); - if (elasticsearchVersion == ElasticsearchVersion.ES2 - || elasticsearchVersion == ElasticsearchVersion.ES5) { - if (type == null || "".equals(type)) { - type = DEFAULT_TYPE; - } - return new RequiredIndexTypeSerializer(type); - } - if (elasticsearchVersion == ElasticsearchVersion.ES6) { - if (type != null && !"".equals(type)) { - return new RequiredIndexTypeSerializer(type); - } - } - return new NotIndexTypeSerializer(); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/impl/NotIndexTypeSerializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/impl/NotIndexTypeSerializer.java deleted file mode 100644 index cdab29625e6..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/impl/NotIndexTypeSerializer.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type.impl; - -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type.IndexTypeSerializer; - -import java.util.Map; - -/** not need an index type for elasticsearch version:6.*,7.*,8.* */ -public class NotIndexTypeSerializer implements IndexTypeSerializer { - - @Override - public void fillType(Map indexInner) {} -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/impl/RequiredIndexTypeSerializer.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/impl/RequiredIndexTypeSerializer.java deleted file mode 100644 index 53c63ba840f..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/serialize/type/impl/RequiredIndexTypeSerializer.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type.impl; - -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.type.IndexTypeSerializer; - -import java.util.Map; - -/** generate an index type for elasticsearch version:2.*,5.*,6.* */ -public class RequiredIndexTypeSerializer implements IndexTypeSerializer { - - private final String type; - - public RequiredIndexTypeSerializer(String type) { - this.type = type; - } - - @Override - public void fillType(Map indexInner) { - indexInner.put("_type", type); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSink.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSink.java deleted file mode 100644 index e869871683a..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSink.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.sink; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.apache.seatunnel.api.common.PrepareFailException; -import org.apache.seatunnel.api.sink.SeaTunnelSink; -import org.apache.seatunnel.api.sink.SinkWriter; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.SinkConfig; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.state.ElasticsearchAggregatedCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.state.ElasticsearchCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.state.ElasticsearchSinkState; - -import com.google.auto.service.AutoService; - -@AutoService(SeaTunnelSink.class) -public class ElasticsearchSink - implements SeaTunnelSink< - SeaTunnelRow, - ElasticsearchSinkState, - ElasticsearchCommitInfo, - ElasticsearchAggregatedCommitInfo> { - - private Config pluginConfig; - private SeaTunnelRowType seaTunnelRowType; - - private int maxBatchSize = SinkConfig.MAX_BATCH_SIZE.defaultValue(); - - private int maxRetryCount = SinkConfig.MAX_RETRY_COUNT.defaultValue(); - - @Override - public String getPluginName() { - return "Elasticsearch"; - } - - @Override - public void prepare(Config pluginConfig) throws PrepareFailException { - this.pluginConfig = pluginConfig; - if (pluginConfig.hasPath(SinkConfig.MAX_BATCH_SIZE.key())) { - maxBatchSize = pluginConfig.getInt(SinkConfig.MAX_BATCH_SIZE.key()); - } - if (pluginConfig.hasPath(SinkConfig.MAX_RETRY_COUNT.key())) { - maxRetryCount = pluginConfig.getInt(SinkConfig.MAX_RETRY_COUNT.key()); - } - } - - @Override - public void setTypeInfo(SeaTunnelRowType seaTunnelRowType) { - this.seaTunnelRowType = seaTunnelRowType; - } - - @Override - public SinkWriter createWriter( - SinkWriter.Context context) { - return new ElasticsearchSinkWriter( - context, seaTunnelRowType, pluginConfig, maxBatchSize, maxRetryCount); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSinkFactory.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSinkFactory.java deleted file mode 100644 index 5678380e2ac..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSinkFactory.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.sink; - -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.table.factory.Factory; -import org.apache.seatunnel.api.table.factory.TableSinkFactory; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.EsClusterConnectionConfig; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.SinkConfig; - -import com.google.auto.service.AutoService; - -@AutoService(Factory.class) -public class ElasticsearchSinkFactory implements TableSinkFactory { - @Override - public String factoryIdentifier() { - return "Elasticsearch-hw"; - } - - @Override - public OptionRule optionRule() { - return OptionRule.builder() - .required(EsClusterConnectionConfig.HOSTS, SinkConfig.INDEX) - .optional( - SinkConfig.INDEX_TYPE, - SinkConfig.PRIMARY_KEYS, - SinkConfig.KEY_DELIMITER, - EsClusterConnectionConfig.USERNAME, - EsClusterConnectionConfig.PASSWORD, - SinkConfig.MAX_RETRY_COUNT, - SinkConfig.MAX_BATCH_SIZE, - EsClusterConnectionConfig.TLS_VERIFY_CERTIFICATE, - EsClusterConnectionConfig.TLS_VERIFY_HOSTNAME, - EsClusterConnectionConfig.TLS_KEY_STORE_PATH, - EsClusterConnectionConfig.TLS_KEY_STORE_PASSWORD, - EsClusterConnectionConfig.TLS_TRUST_STORE_PATH, - EsClusterConnectionConfig.TLS_TRUST_STORE_PASSWORD) - .build(); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSinkWriter.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSinkWriter.java deleted file mode 100644 index 33cbefbc1cd..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/sink/ElasticsearchSinkWriter.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.sink; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.apache.seatunnel.api.sink.SinkWriter; -import org.apache.seatunnel.api.table.type.RowKind; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; -import org.apache.seatunnel.common.utils.RetryUtils; -import org.apache.seatunnel.common.utils.RetryUtils.RetryMaterial; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.client.EsRestClient; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.BulkResponse; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.IndexInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorException; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.ElasticsearchRowSerializer; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.SeaTunnelRowSerializer; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.state.ElasticsearchCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.state.ElasticsearchSinkState; - -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -/** - * ElasticsearchSinkWriter is a sink writer that will write {@link SeaTunnelRow} to Elasticsearch. - */ -@Slf4j -public class ElasticsearchSinkWriter - implements SinkWriter { - - private final SinkWriter.Context context; - - private final int maxBatchSize; - - private final SeaTunnelRowSerializer seaTunnelRowSerializer; - private final List requestEsList; - private EsRestClient esRestClient; - private RetryMaterial retryMaterial; - private static final long DEFAULT_SLEEP_TIME_MS = 200L; - - public ElasticsearchSinkWriter( - SinkWriter.Context context, - SeaTunnelRowType seaTunnelRowType, - Config pluginConfig, - int maxBatchSize, - int maxRetryCount) { - this.context = context; - this.maxBatchSize = maxBatchSize; - - IndexInfo indexInfo = new IndexInfo(pluginConfig); - esRestClient = EsRestClient.createInstance(pluginConfig); - this.seaTunnelRowSerializer = - new ElasticsearchRowSerializer( - esRestClient.getClusterInfo(), indexInfo, seaTunnelRowType); - - this.requestEsList = new ArrayList<>(maxBatchSize); - this.retryMaterial = - new RetryMaterial(maxRetryCount, true, exception -> true, DEFAULT_SLEEP_TIME_MS); - } - - @Override - public void write(SeaTunnelRow element) { - if (RowKind.UPDATE_BEFORE.equals(element.getRowKind())) { - return; - } - - String indexRequestRow = seaTunnelRowSerializer.serializeRow(element); - requestEsList.add(indexRequestRow); - if (requestEsList.size() >= maxBatchSize) { - bulkEsWithRetry(this.esRestClient, this.requestEsList); - } - } - - @Override - public Optional prepareCommit() { - bulkEsWithRetry(this.esRestClient, this.requestEsList); - return Optional.empty(); - } - - @Override - public void abortPrepare() {} - - public synchronized void bulkEsWithRetry( - EsRestClient esRestClient, List requestEsList) { - try { - RetryUtils.retryWithException( - () -> { - if (requestEsList.size() > 0) { - String requestBody = String.join("\n", requestEsList) + "\n"; - BulkResponse bulkResponse = esRestClient.bulk(requestBody); - if (bulkResponse.isErrors()) { - throw new ElasticsearchConnectorException( - ElasticsearchConnectorErrorCode.BULK_RESPONSE_ERROR, - "bulk es error: " + bulkResponse.getResponse()); - } - return bulkResponse; - } - return null; - }, - retryMaterial); - requestEsList.clear(); - } catch (Exception e) { - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.SQL_OPERATION_FAILED, - "ElasticSearch execute batch statement error", - e); - } - } - - @Override - public void close() throws IOException { - bulkEsWithRetry(this.esRestClient, this.requestEsList); - esRestClient.close(); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSource.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSource.java deleted file mode 100644 index 039c00a8360..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSource.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.source; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.apache.seatunnel.api.common.PrepareFailException; -import org.apache.seatunnel.api.source.Boundedness; -import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.source.SourceReader; -import org.apache.seatunnel.api.source.SourceSplitEnumerator; -import org.apache.seatunnel.api.source.SupportColumnProjection; -import org.apache.seatunnel.api.source.SupportParallelism; -import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; -import org.apache.seatunnel.api.table.catalog.schema.TableSchemaOptions; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.catalog.ElasticSearchDataTypeConvertor; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.client.EsRestClient; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.SourceConfig; - -import com.google.auto.service.AutoService; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -@AutoService(SeaTunnelSource.class) -public class ElasticsearchSource - implements SeaTunnelSource< - SeaTunnelRow, ElasticsearchSourceSplit, ElasticsearchSourceState>, - SupportParallelism, - SupportColumnProjection { - - private Config pluginConfig; - - private SeaTunnelRowType rowTypeInfo; - - private List source; - - @Override - public String getPluginName() { - return "Elasticsearch"; - } - - @Override - public void prepare(Config pluginConfig) throws PrepareFailException { - this.pluginConfig = pluginConfig; - if (pluginConfig.hasPath(TableSchemaOptions.SCHEMA.key())) { - // todo: We need to remove the schema in ES. - rowTypeInfo = CatalogTableUtil.buildWithConfig(pluginConfig).getSeaTunnelRowType(); - source = Arrays.asList(rowTypeInfo.getFieldNames()); - } else { - source = pluginConfig.getStringList(SourceConfig.SOURCE.key()); - EsRestClient esRestClient = EsRestClient.createInstance(this.pluginConfig); - Map esFieldType = - esRestClient.getFieldTypeMapping( - pluginConfig.getString(SourceConfig.INDEX.key()), source); - esRestClient.close(); - SeaTunnelDataType[] fieldTypes = new SeaTunnelDataType[source.size()]; - ElasticSearchDataTypeConvertor elasticSearchDataTypeConvertor = - new ElasticSearchDataTypeConvertor(); - for (int i = 0; i < source.size(); i++) { - String esType = esFieldType.get(source.get(i)); - SeaTunnelDataType seaTunnelDataType = - elasticSearchDataTypeConvertor.toSeaTunnelType(source.get(i), esType); - fieldTypes[i] = seaTunnelDataType; - } - rowTypeInfo = new SeaTunnelRowType(source.toArray(new String[0]), fieldTypes); - } - } - - @Override - public Boundedness getBoundedness() { - return Boundedness.BOUNDED; - } - - @Override - public SeaTunnelDataType getProducedType() { - return this.rowTypeInfo; - } - - @Override - public SourceReader createReader( - SourceReader.Context readerContext) { - return new ElasticsearchSourceReader(readerContext, pluginConfig, rowTypeInfo); - } - - @Override - public SourceSplitEnumerator - createEnumerator( - SourceSplitEnumerator.Context enumeratorContext) { - return new ElasticsearchSourceSplitEnumerator(enumeratorContext, pluginConfig, source); - } - - @Override - public SourceSplitEnumerator - restoreEnumerator( - SourceSplitEnumerator.Context enumeratorContext, - ElasticsearchSourceState sourceState) { - return new ElasticsearchSourceSplitEnumerator( - enumeratorContext, sourceState, pluginConfig, source); - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceFactory.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceFactory.java deleted file mode 100644 index 933ff23420c..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceFactory.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.source; - -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.table.catalog.schema.TableSchemaOptions; -import org.apache.seatunnel.api.table.factory.Factory; -import org.apache.seatunnel.api.table.factory.TableSourceFactory; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.EsClusterConnectionConfig; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.SourceConfig; - -import com.google.auto.service.AutoService; - -@AutoService(Factory.class) -public class ElasticsearchSourceFactory implements TableSourceFactory { - @Override - public String factoryIdentifier() { - return "Elasticsearch-hw"; - } - - @Override - public OptionRule optionRule() { - return OptionRule.builder() - .required(EsClusterConnectionConfig.HOSTS, SourceConfig.INDEX) - .optional( - EsClusterConnectionConfig.USERNAME, - EsClusterConnectionConfig.PASSWORD, - SourceConfig.SCROLL_TIME, - SourceConfig.SCROLL_SIZE, - SourceConfig.QUERY, - EsClusterConnectionConfig.TLS_VERIFY_CERTIFICATE, - EsClusterConnectionConfig.TLS_VERIFY_HOSTNAME, - EsClusterConnectionConfig.TLS_KEY_STORE_PATH, - EsClusterConnectionConfig.TLS_KEY_STORE_PASSWORD, - EsClusterConnectionConfig.TLS_TRUST_STORE_PATH, - EsClusterConnectionConfig.TLS_TRUST_STORE_PASSWORD) - .exclusive(SourceConfig.SOURCE, TableSchemaOptions.SCHEMA) - .build(); - } - - @Override - public Class getSourceClass() { - return ElasticsearchSource.class; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceReader.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceReader.java deleted file mode 100644 index a15d4e3e734..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceReader.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.source; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.apache.seatunnel.api.source.Collector; -import org.apache.seatunnel.api.source.SourceReader; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.client.EsRestClient; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source.ScrollResult; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source.SourceIndexInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.source.DefaultSeaTunnelRowDeserializer; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.source.ElasticsearchRecord; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.serialize.source.SeaTunnelRowDeserializer; - -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Deque; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -@Slf4j -public class ElasticsearchSourceReader - implements SourceReader { - - SourceReader.Context context; - - private Config pluginConfig; - - private EsRestClient esRestClient; - - private final SeaTunnelRowDeserializer deserializer; - - Deque splits = new LinkedList<>(); - boolean noMoreSplit; - - private final long pollNextWaitTime = 1000L; - - public ElasticsearchSourceReader( - SourceReader.Context context, Config pluginConfig, SeaTunnelRowType rowTypeInfo) { - this.context = context; - this.pluginConfig = pluginConfig; - this.deserializer = new DefaultSeaTunnelRowDeserializer(rowTypeInfo); - } - - @Override - public void open() { - esRestClient = EsRestClient.createInstance(this.pluginConfig); - } - - @Override - public void close() throws IOException { - esRestClient.close(); - } - - @Override - public void pollNext(Collector output) throws Exception { - synchronized (output.getCheckpointLock()) { - ElasticsearchSourceSplit split = splits.poll(); - if (split != null) { - SourceIndexInfo sourceIndexInfo = split.getSourceIndexInfo(); - ScrollResult scrollResult = - esRestClient.searchByScroll( - sourceIndexInfo.getIndex(), - sourceIndexInfo.getSource(), - sourceIndexInfo.getQuery(), - sourceIndexInfo.getScrollTime(), - sourceIndexInfo.getScrollSize()); - outputFromScrollResult(scrollResult, sourceIndexInfo.getSource(), output); - while (scrollResult.getDocs() != null && scrollResult.getDocs().size() > 0) { - scrollResult = - esRestClient.searchWithScrollId( - scrollResult.getScrollId(), sourceIndexInfo.getScrollTime()); - outputFromScrollResult(scrollResult, sourceIndexInfo.getSource(), output); - } - } else if (noMoreSplit) { - // signal to the source that we have reached the end of the data. - log.info("Closed the bounded ELasticsearch source"); - context.signalNoMoreElement(); - } else { - Thread.sleep(pollNextWaitTime); - } - } - } - - private void outputFromScrollResult( - ScrollResult scrollResult, List source, Collector output) { - for (Map doc : scrollResult.getDocs()) { - SeaTunnelRow seaTunnelRow = - deserializer.deserialize(new ElasticsearchRecord(doc, source)); - output.collect(seaTunnelRow); - } - } - - @Override - public List snapshotState(long checkpointId) throws Exception { - return new ArrayList<>(splits); - } - - @Override - public void addSplits(List splits) { - this.splits.addAll(splits); - } - - @Override - public void handleNoMoreSplits() { - noMoreSplit = true; - } - - @Override - public void notifyCheckpointComplete(long checkpointId) throws Exception {} -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceSplit.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceSplit.java deleted file mode 100644 index 28d6f1086f5..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceSplit.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.source; - -import org.apache.seatunnel.api.source.SourceSplit; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source.SourceIndexInfo; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; - -@ToString -@AllArgsConstructor -public class ElasticsearchSourceSplit implements SourceSplit { - - private static final long serialVersionUID = -1L; - - private String splitId; - - @Getter private SourceIndexInfo sourceIndexInfo; - - @Override - public String splitId() { - return splitId; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceSplitEnumerator.java deleted file mode 100644 index 9b307683dbe..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceSplitEnumerator.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.source; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.apache.seatunnel.api.source.SourceSplitEnumerator; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.client.EsRestClient; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.config.SourceConfig; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source.IndexDocsCount; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.source.SourceIndexInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.exception.ElasticsearchConnectorException; - -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -@Slf4j -public class ElasticsearchSourceSplitEnumerator - implements SourceSplitEnumerator { - - private SourceSplitEnumerator.Context context; - - private Config pluginConfig; - - private EsRestClient esRestClient; - - private final Object stateLock = new Object(); - - private Map> pendingSplit; - - private List source; - - private volatile boolean shouldEnumerate; - - public ElasticsearchSourceSplitEnumerator( - SourceSplitEnumerator.Context context, - Config pluginConfig, - List source) { - this(context, null, pluginConfig, source); - } - - public ElasticsearchSourceSplitEnumerator( - SourceSplitEnumerator.Context context, - ElasticsearchSourceState sourceState, - Config pluginConfig, - List source) { - this.context = context; - this.pluginConfig = pluginConfig; - this.pendingSplit = new HashMap<>(); - this.shouldEnumerate = sourceState == null; - if (sourceState != null) { - this.shouldEnumerate = sourceState.isShouldEnumerate(); - this.pendingSplit.putAll(sourceState.getPendingSplit()); - } - this.source = source; - } - - @Override - public void open() { - esRestClient = EsRestClient.createInstance(pluginConfig); - } - - @Override - public void run() { - Set readers = context.registeredReaders(); - if (shouldEnumerate) { - List newSplits = getElasticsearchSplit(); - - synchronized (stateLock) { - addPendingSplit(newSplits); - shouldEnumerate = false; - } - - assignSplit(readers); - } - - log.debug( - "No more splits to assign." + " Sending NoMoreSplitsEvent to reader {}.", readers); - readers.forEach(context::signalNoMoreSplits); - } - - private void addPendingSplit(Collection splits) { - int readerCount = context.currentParallelism(); - for (ElasticsearchSourceSplit split : splits) { - int ownerReader = getSplitOwner(split.splitId(), readerCount); - log.info("Assigning {} to {} reader.", split, ownerReader); - pendingSplit.computeIfAbsent(ownerReader, r -> new ArrayList<>()).add(split); - } - } - - private static int getSplitOwner(String tp, int numReaders) { - return (tp.hashCode() & Integer.MAX_VALUE) % numReaders; - } - - private void assignSplit(Collection readers) { - log.debug("Assign pendingSplits to readers {}", readers); - - for (int reader : readers) { - List assignmentForReader = pendingSplit.remove(reader); - if (assignmentForReader != null && !assignmentForReader.isEmpty()) { - log.info("Assign splits {} to reader {}", assignmentForReader, reader); - try { - context.assignSplit(reader, assignmentForReader); - } catch (Exception e) { - log.error( - "Failed to assign splits {} to reader {}", - assignmentForReader, - reader, - e); - pendingSplit.put(reader, assignmentForReader); - } - } - } - } - - private List getElasticsearchSplit() { - List splits = new ArrayList<>(); - String scrollTime = SourceConfig.SCROLL_TIME.defaultValue(); - if (pluginConfig.hasPath(SourceConfig.SCROLL_TIME.key())) { - scrollTime = pluginConfig.getString(SourceConfig.SCROLL_TIME.key()); - } - int scrollSize = SourceConfig.SCROLL_SIZE.defaultValue(); - if (pluginConfig.hasPath(SourceConfig.SCROLL_SIZE.key())) { - scrollSize = pluginConfig.getInt(SourceConfig.SCROLL_SIZE.key()); - } - Map query = SourceConfig.QUERY.defaultValue(); - if (pluginConfig.hasPath(SourceConfig.QUERY.key())) { - query = (Map) pluginConfig.getAnyRef(SourceConfig.QUERY.key()); - } - - List indexDocsCounts = - esRestClient.getIndexDocsCount(pluginConfig.getString(SourceConfig.INDEX.key())); - indexDocsCounts = - indexDocsCounts.stream() - .filter(x -> x.getDocsCount() != null && x.getDocsCount() > 0) - .sorted(Comparator.comparingLong(IndexDocsCount::getDocsCount)) - .collect(Collectors.toList()); - for (IndexDocsCount indexDocsCount : indexDocsCounts) { - splits.add( - new ElasticsearchSourceSplit( - String.valueOf(indexDocsCount.getIndex().hashCode()), - new SourceIndexInfo( - indexDocsCount.getIndex(), - source, - query, - scrollTime, - scrollSize))); - } - return splits; - } - - @Override - public void close() throws IOException { - esRestClient.close(); - } - - @Override - public void addSplitsBack(List splits, int subtaskId) { - if (!splits.isEmpty()) { - addPendingSplit(splits); - assignSplit(Collections.singletonList(subtaskId)); - } - } - - @Override - public int currentUnassignedSplitSize() { - return pendingSplit.size(); - } - - @Override - public void handleSplitRequest(int subtaskId) { - throw new ElasticsearchConnectorException( - CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION, - "Unsupported handleSplitRequest: " + subtaskId); - } - - @Override - public void registerReader(int subtaskId) { - log.debug("Register reader {} to IoTDBSourceSplitEnumerator.", subtaskId); - if (!pendingSplit.isEmpty()) { - assignSplit(Collections.singletonList(subtaskId)); - } - } - - @Override - public ElasticsearchSourceState snapshotState(long checkpointId) throws Exception { - synchronized (stateLock) { - return new ElasticsearchSourceState(shouldEnumerate, pendingSplit); - } - } - - @Override - public void notifyCheckpointComplete(long checkpointId) throws Exception {} -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceState.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceState.java deleted file mode 100644 index e6ffe6343ba..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/source/ElasticsearchSourceState.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.source; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -@AllArgsConstructor -@Getter -public class ElasticsearchSourceState implements Serializable { - private boolean shouldEnumerate; - private Map> pendingSplit; -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchAggregatedCommitInfo.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchAggregatedCommitInfo.java deleted file mode 100644 index 3e893750ca6..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchAggregatedCommitInfo.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.state; - -import java.io.Serializable; - -/** - * Right now, we don't need aggregated commit in kafka. Todo: we need to add a default - * implementation of this state. - */ -public class ElasticsearchAggregatedCommitInfo implements Serializable {} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchCommitInfo.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchCommitInfo.java deleted file mode 100644 index 2e5dc2b41ab..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchCommitInfo.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.state; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.io.Serializable; -import java.util.Properties; - -@Data -@AllArgsConstructor -public class ElasticsearchCommitInfo implements Serializable { - - private final String transactionId; - private final Properties kafkaProperties; - private final long producerId; - private final short epoch; -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchSinkState.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchSinkState.java deleted file mode 100644 index 443df9c70a6..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/state/ElasticsearchSinkState.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.state; - -import java.io.Serializable; - -public class ElasticsearchSinkState implements Serializable {} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/util/RegexUtils.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/util/RegexUtils.java deleted file mode 100644 index 4ca02d40ddb..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/util/RegexUtils.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.util; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class RegexUtils { - - public static List extractDatas(String content, String regex) { - List datas = new ArrayList<>(); - Pattern pattern = Pattern.compile(regex, Pattern.DOTALL); - Matcher matcher = pattern.matcher(content); - while (matcher.find()) { - String result = matcher.group(1); - datas.add(result); - } - return datas; - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/util/SSLUtils.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/util/SSLUtils.java deleted file mode 100644 index f9da7761bbf..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/hw/util/SSLUtils.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.util; - -import io.airlift.security.pem.PemReader; - -import javax.net.ssl.KeyManager; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.net.ssl.X509TrustManager; -import javax.security.auth.x500.X500Principal; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.security.cert.Certificate; -import java.security.cert.CertificateExpiredException; -import java.security.cert.CertificateNotYetValidException; -import java.security.cert.X509Certificate; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -import static java.util.Collections.list; - -@SuppressWarnings("MagicNumber") -public final class SSLUtils { - - public static Optional buildSSLContext( - Optional keyStorePath, - Optional keyStorePassword, - Optional trustStorePath, - Optional trustStorePassword) - throws GeneralSecurityException, IOException { - if (!keyStorePath.isPresent() && !trustStorePath.isPresent()) { - return Optional.empty(); - } - return Optional.of( - createSSLContext( - keyStorePath, keyStorePassword, trustStorePath, trustStorePassword)); - } - - private static SSLContext createSSLContext( - Optional keyStorePath, - Optional keyStorePassword, - Optional trustStorePath, - Optional trustStorePassword) - throws GeneralSecurityException, IOException { - // load KeyStore if configured and get KeyManagers - KeyStore keyStore = null; - KeyManager[] keyManagers = null; - if (keyStorePath.isPresent()) { - File keyStoreFile = new File(keyStorePath.get()); - char[] keyManagerPassword; - try { - // attempt to read the key store as a PEM file - keyStore = PemReader.loadKeyStore(keyStoreFile, keyStoreFile, keyStorePassword); - // for PEM encoded keys, the password is used to decrypt the specific key (and does - // not protect the keystore itself) - keyManagerPassword = new char[0]; - } catch (IOException | GeneralSecurityException ignored) { - keyManagerPassword = keyStorePassword.map(String::toCharArray).orElse(null); - - keyStore = KeyStore.getInstance(KeyStore.getDefaultType()); - try (InputStream in = new FileInputStream(keyStoreFile)) { - keyStore.load(in, keyManagerPassword); - } - } - validateCertificates(keyStore); - KeyManagerFactory keyManagerFactory = - KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStore, keyManagerPassword); - keyManagers = keyManagerFactory.getKeyManagers(); - } - - // load TrustStore if configured, otherwise use KeyStore - KeyStore trustStore = keyStore; - if (trustStorePath.isPresent()) { - File trustStoreFile = new File(trustStorePath.get()); - trustStore = loadTrustStore(trustStoreFile, trustStorePassword); - } - - // create TrustManagerFactory - TrustManagerFactory trustManagerFactory = - TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(trustStore); - - // get X509TrustManager - TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); - if (trustManagers.length != 1 || !(trustManagers[0] instanceof X509TrustManager)) { - throw new RuntimeException( - "Unexpected default trust managers:" + Arrays.toString(trustManagers)); - } - // create SSLContext - SSLContext result = SSLContext.getInstance("SSL"); - result.init(keyManagers, trustManagers, null); - return result; - } - - private static KeyStore loadTrustStore(File trustStorePath, Optional trustStorePassword) - throws IOException, GeneralSecurityException { - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); - try { - // attempt to read the trust store as a PEM file - List certificateChain = PemReader.readCertificateChain(trustStorePath); - if (!certificateChain.isEmpty()) { - trustStore.load(null, null); - for (X509Certificate certificate : certificateChain) { - X500Principal principal = certificate.getSubjectX500Principal(); - trustStore.setCertificateEntry(principal.getName(), certificate); - } - return trustStore; - } - } catch (IOException | GeneralSecurityException ignored) { - // ignored - } - - try (InputStream in = new FileInputStream(trustStorePath)) { - trustStore.load(in, trustStorePassword.map(String::toCharArray).orElse(null)); - } - return trustStore; - } - - private static void validateCertificates(KeyStore keyStore) throws GeneralSecurityException { - for (String alias : list(keyStore.aliases())) { - if (!keyStore.isKeyEntry(alias)) { - continue; - } - Certificate certificate = keyStore.getCertificate(alias); - if (!(certificate instanceof X509Certificate)) { - continue; - } - - try { - ((X509Certificate) certificate).checkValidity(); - } catch (CertificateExpiredException e) { - throw new CertificateExpiredException( - "KeyStore certificate is expired: " + e.getMessage()); - } catch (CertificateNotYetValidException e) { - throw new CertificateNotYetValidException( - "KeyStore certificate is not yet valid: " + e.getMessage()); - } - } - } -} diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/resources/conf/esParams.properties b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/resources/conf/esParams.properties deleted file mode 100644 index 54d2299ea89..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/resources/conf/esParams.properties +++ /dev/null @@ -1,12 +0,0 @@ -esServerHost=10.68.120.90:24100,10.68.120.90:24100,10.68.120.90:24100 -connectTimeout=5000 -socketTimeout=60000 -connectionRequestTimeout=3000 -maxConnPerRoute=100 -maxConnTotal=1000 -isSecureMode=true -#principal????? -principal=esuser -snifferEnable=true -# jaas.conf???????????? -customJaasPath= diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/resources/conf/krb5.conf b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/resources/conf/krb5.conf deleted file mode 100644 index d83fe424cd3..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/main/resources/conf/krb5.conf +++ /dev/null @@ -1,47 +0,0 @@ -[kdcdefaults] -kdc_ports = 10.68.120.91:21732 -kdc_tcp_ports = "" - -[libdefaults] -default_realm = HADOOP.COM -kdc_timeout = 2500 -clockskew = 300 -use_dns_lookup = 0 -udp_preference_limit = 1465 -max_retries = 5 -dns_lookup_kdc = false -dns_lookup_realm = false -renewable = false -forwardable = false -renew_lifetime = 0m -max_renewable_life = 30m -allow_extend_version = false -default_ccache_name = FILE:/tmp//krb5cc_%{uid} - -[realms] -HADOOP.COM = { -kdc = 10.68.120.90:21732 -kdc = 10.68.120.91:21732 -admin_server = 10.68.120.90:21730 -admin_server = 10.68.120.91:21730 -kpasswd_server = 10.68.120.90:21731 -kpasswd_server = 10.68.120.91:21731 -kpasswd_port = 21731 -kadmind_port = 21730 -kadmind_listen = 10.68.120.91:21730 -kpasswd_listen = 10.68.120.91:21731 -renewable = false -forwardable = false -renew_lifetime = 0m -max_renewable_life = 30m -acl_file = /opt/huawei/Bigdata/FusionInsight_BASE_6.5.1.3/install/FusionInsight-kerberos-1.17/kerberos/var/krb5kdc/kadm5.acl -key_stash_file = /opt/huawei/Bigdata/FusionInsight_BASE_6.5.1.3/install/FusionInsight-kerberos-1.17/kerberos/var/krb5kdc/.k5.HADOOP.COM -} - -[domain_realm] -.hadoop.com = HADOOP.COM - -[logging] -kdc = SYSLOG:INFO:DAEMON -admin_server = SYSLOG:INFO:DAEMON -default = SYSLOG:NOTICE:DAEMON diff --git a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/ElasticsearchFactoryTest.java b/seatunnel-connectors-v2/connector-elasticsearch-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/ElasticsearchFactoryTest.java deleted file mode 100644 index b0132e9fcd7..00000000000 --- a/seatunnel-connectors-v2/connector-elasticsearch-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/ElasticsearchFactoryTest.java +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.elasticsearch; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigMemorySize; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigMergeable; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigOrigin; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigResolveOptions; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; - -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.client.EsRestClient; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.dto.ElasticsearchClusterInfo; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.sink.ElasticsearchSinkFactory; -import org.apache.seatunnel.connectors.seatunnel.elasticsearch.hw.source.ElasticsearchSourceFactory; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.time.Duration; -import java.time.Period; -import java.time.temporal.TemporalAmount; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -public class ElasticsearchFactoryTest { - - @Test - void optionRule() { - Assertions.assertNotNull((new ElasticsearchSourceFactory()).optionRule()); - Assertions.assertNotNull((new ElasticsearchSinkFactory()).optionRule()); - } - - @Test - void getConnTest() { - EsRestClient instance = - EsRestClient.createInstance( - new Config() { - @Override - public ConfigObject root() { - return null; - } - - @Override - public ConfigOrigin origin() { - return null; - } - - @Override - public Config withFallback(ConfigMergeable configMergeable) { - return null; - } - - @Override - public Config resolve() { - return null; - } - - @Override - public Config resolve(ConfigResolveOptions configResolveOptions) { - return null; - } - - @Override - public boolean isResolved() { - return false; - } - - @Override - public Config resolveWith(Config config) { - return null; - } - - @Override - public Config resolveWith( - Config config, ConfigResolveOptions configResolveOptions) { - return null; - } - - @Override - public void checkValid(Config config, String... strings) {} - - @Override - public boolean hasPath(String s) { - return false; - } - - @Override - public boolean hasPathOrNull(String s) { - return false; - } - - @Override - public boolean isEmpty() { - return false; - } - - @Override - public Set> entrySet() { - return null; - } - - @Override - public boolean getIsNull(String s) { - return false; - } - - @Override - public boolean getBoolean(String s) { - return false; - } - - @Override - public Number getNumber(String s) { - return null; - } - - @Override - public int getInt(String s) { - return 0; - } - - @Override - public long getLong(String s) { - return 0; - } - - @Override - public double getDouble(String s) { - return 0; - } - - @Override - public String getString(String s) { - return null; - } - - @Override - public > T getEnum(Class aClass, String s) { - return null; - } - - @Override - public ConfigObject getObject(String s) { - return null; - } - - @Override - public Config getConfig(String s) { - return null; - } - - @Override - public Object getAnyRef(String s) { - return null; - } - - @Override - public ConfigValue getValue(String s) { - return null; - } - - @Override - public Long getBytes(String s) { - return null; - } - - @Override - public ConfigMemorySize getMemorySize(String s) { - return null; - } - - @Override - public Long getMilliseconds(String s) { - return null; - } - - @Override - public Long getNanoseconds(String s) { - return null; - } - - @Override - public long getDuration(String s, TimeUnit timeUnit) { - return 0; - } - - @Override - public Duration getDuration(String s) { - return null; - } - - @Override - public Period getPeriod(String s) { - return null; - } - - @Override - public TemporalAmount getTemporal(String s) { - return null; - } - - @Override - public ConfigList getList(String s) { - return null; - } - - @Override - public List getBooleanList(String s) { - return null; - } - - @Override - public List getNumberList(String s) { - return null; - } - - @Override - public List getIntList(String s) { - return null; - } - - @Override - public List getLongList(String s) { - return null; - } - - @Override - public List getDoubleList(String s) { - return null; - } - - @Override - public List getStringList(String s) { - return null; - } - - @Override - public > List getEnumList( - Class aClass, String s) { - return null; - } - - @Override - public List getObjectList(String s) { - return null; - } - - @Override - public List getConfigList(String s) { - return null; - } - - @Override - public List getAnyRefList(String s) { - return null; - } - - @Override - public List getBytesList(String s) { - return null; - } - - @Override - public List getMemorySizeList(String s) { - return null; - } - - @Override - public List getMillisecondsList(String s) { - return null; - } - - @Override - public List getNanosecondsList(String s) { - return null; - } - - @Override - public List getDurationList(String s, TimeUnit timeUnit) { - return null; - } - - @Override - public List getDurationList(String s) { - return null; - } - - @Override - public Config withOnlyPath(String s) { - return null; - } - - @Override - public Config withoutPath(String s) { - return null; - } - - @Override - public Config atPath(String s) { - return null; - } - - @Override - public Config atKey(String s) { - return null; - } - - @Override - public Config withValue(String s, ConfigValue configValue) { - return null; - } - }); - ElasticsearchClusterInfo clusterInfo = instance.getClusterInfo(); - System.out.println(clusterInfo); - // instance.createIndex("huawei"); - // instance.dropIndex("huawei"); - List index = instance.listIndex(); - // index.stream().forEach(System.out::println); - // List huawei = instance.getIndexDocsCount("event20221121"); - // huawei.stream().forEach(System.out::println); - Map fieldTypeMapping = instance.getFieldTypeMapping("event20221005", null); - System.out.println(fieldTypeMapping.toString()); - } -} diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/hive/HiveJdbcConnectionProvider.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/hive/HiveJdbcConnectionProvider.java index 1a45a8600a9..131ebe2d400 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/hive/HiveJdbcConnectionProvider.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/dialect/hive/HiveJdbcConnectionProvider.java @@ -14,14 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.seatunnel.connectors.seatunnel.jdbc.internal.dialect.hive; import org.apache.seatunnel.connectors.seatunnel.jdbc.config.JdbcConnectionConfig; import org.apache.seatunnel.connectors.seatunnel.jdbc.exception.JdbcConnectorErrorCode; import org.apache.seatunnel.connectors.seatunnel.jdbc.exception.JdbcConnectorException; import org.apache.seatunnel.connectors.seatunnel.jdbc.internal.connection.SimpleJdbcConnectionProvider; +import org.apache.seatunnel.connectors.seatunnel.jdbc.utils.LoginUtil; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.security.UserGroupInformation; import lombok.NonNull; @@ -50,8 +54,15 @@ public Connection getOrEstablishConnection() throws SQLException, ClassNotFoundE System.setProperty("java.security.krb5.conf", jdbcConfig.krb5Path); Configuration configuration = new Configuration(); configuration.set("hadoop.security.authentication", "kerberos"); - UserGroupInformation.setConfiguration(configuration); + configuration.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true"); + try { + if (StringUtils.isNotBlank("hiveserver2")) { + LoginUtil.setJaasConf( + "Client", "hiveuser@HADOOP.COM", jdbcConfig.kerberosKeytabPath); + LoginUtil.setZookeeperServerPrincipal("zookeeper/hadoop.hadoop.com"); + } + UserGroupInformation.setConfiguration(configuration); UserGroupInformation.loginUserFromKeytab( jdbcConfig.kerberosPrincipal, jdbcConfig.kerberosKeytabPath); } catch (IOException e) { diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/LoginUtil.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/LoginUtil.java new file mode 100644 index 00000000000..99f5ad59e24 --- /dev/null +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/utils/LoginUtil.java @@ -0,0 +1,461 @@ +package org.apache.seatunnel.connectors.seatunnel.jdbc.utils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosUtil; +import org.apache.log4j.Logger; + +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class LoginUtil { + + private static final Logger LOG = Logger.getLogger(LoginUtil.class); + + private static final String JAVA_SECURITY_KRB5_CONF_KEY = "java.security.krb5.conf"; + + private static final String ZOOKEEPER_SERVER_PRINCIPAL_KEY = "zookeeper.server.principal"; + + private static final String LOGIN_FAILED_CAUSE_PASSWORD_WRONG = + "(wrong password) keytab file and user not match, you can kinit -k -t keytab user in client server to check"; + + private static final String LOGIN_FAILED_CAUSE_TIME_WRONG = + "(clock skew) time of local server and remote server not match, please check ntp to remote server"; + + private static final String LOGIN_FAILED_CAUSE_AES256_WRONG = + "(aes256 not support) aes256 not support by default jdk/jre, need copy local_policy.jar and US_export_policy.jar from remote server in path /opt/huawei/Bigdata/jdk/jre/lib/security"; + + private static final String LOGIN_FAILED_CAUSE_PRINCIPAL_WRONG = + "(no rule) principal format not support by default, need add property hadoop.security.auth_to_local(in core-site.xml) value RULE:[1:$1] RULE:[2:$1]"; + + private static final String LOGIN_FAILED_CAUSE_TIME_OUT = + "(time out) can not connect to kdc server or there is fire wall in the network"; + + private static final boolean IS_IBM_JDK = System.getProperty("java.vendor").contains("IBM"); + + public static synchronized void login( + String userPrincipal, String userKeytabPath, String krb5ConfPath, Configuration conf) + throws IOException { + // 1.check input parameters + if ((userPrincipal == null) || (userPrincipal.length() <= 0)) { + LOG.error("input userPrincipal is invalid."); + throw new IOException("input userPrincipal is invalid."); + } + + if ((userKeytabPath == null) || (userKeytabPath.length() <= 0)) { + LOG.error("input userKeytabPath is invalid."); + throw new IOException("input userKeytabPath is invalid."); + } + + if ((krb5ConfPath == null) || (krb5ConfPath.length() <= 0)) { + LOG.error("input krb5ConfPath is invalid."); + throw new IOException("input krb5ConfPath is invalid."); + } + + if ((conf == null)) { + LOG.error("input conf is invalid."); + throw new IOException("input conf is invalid."); + } + + // 2.check file exsits + File userKeytabFile = new File(userKeytabPath); + if (!userKeytabFile.exists()) { + LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit."); + throw new IOException( + "userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit."); + } + if (!userKeytabFile.isFile()) { + LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") is not a file."); + throw new IOException( + "userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") is not a file."); + } + + File krb5ConfFile = new File(krb5ConfPath); + if (!krb5ConfFile.exists()) { + LOG.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") does not exsit."); + throw new IOException( + "krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") does not exsit."); + } + if (!krb5ConfFile.isFile()) { + LOG.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") is not a file."); + throw new IOException( + "krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") is not a file."); + } + + // 3.set and check krb5config + setKrb5Config(krb5ConfFile.getAbsolutePath()); + setConfiguration(conf); + + // 4.login and check for hadoop + loginHadoop(userPrincipal, userKeytabFile.getAbsolutePath()); + + LOG.info("Login success!!!!!!!!!!!!!!"); + } + + private static void setConfiguration(Configuration conf) throws IOException { + UserGroupInformation.setConfiguration(conf); + } + + private static boolean checkNeedLogin(String principal) throws IOException { + if (!UserGroupInformation.isSecurityEnabled()) { + LOG.error( + "UserGroupInformation is not SecurityEnabled, please check if core-site.xml exists in classpath."); + throw new IOException( + "UserGroupInformation is not SecurityEnabled, please check if core-site.xml exists in classpath."); + } + UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); + if ((currentUser != null) && (currentUser.hasKerberosCredentials())) { + if (checkCurrentUserCorrect(principal)) { + LOG.info("current user is " + currentUser + "has logined."); + if (!currentUser.isFromKeytab()) { + LOG.error("current user is not from keytab."); + throw new IOException("current user is not from keytab."); + } + return false; + } else { + LOG.error( + "current user is " + + currentUser + + "has logined. please check your enviroment , especially when it used IBM JDK or kerberos for OS count login!!"); + throw new IOException( + "current user is " + + currentUser + + " has logined. And please check your enviroment!!"); + } + } + + return true; + } + + private static void setKrb5Config(String krb5ConfFile) throws IOException { + System.setProperty(JAVA_SECURITY_KRB5_CONF_KEY, krb5ConfFile); + String ret = System.getProperty(JAVA_SECURITY_KRB5_CONF_KEY); + if (ret == null) { + LOG.error(JAVA_SECURITY_KRB5_CONF_KEY + " is null."); + throw new IOException(JAVA_SECURITY_KRB5_CONF_KEY + " is null."); + } + if (!ret.equals(krb5ConfFile)) { + LOG.error(JAVA_SECURITY_KRB5_CONF_KEY + " is " + ret + " is not " + krb5ConfFile + "."); + throw new IOException( + JAVA_SECURITY_KRB5_CONF_KEY + " is " + ret + " is not " + krb5ConfFile + "."); + } + } + + public static void setJaasConf(String loginContextName, String principal, String keytabFile) + throws IOException { + if ((loginContextName == null) || (loginContextName.length() <= 0)) { + LOG.error("input loginContextName is invalid."); + throw new IOException("input loginContextName is invalid."); + } + + if ((principal == null) || (principal.length() <= 0)) { + LOG.error("input principal is invalid."); + throw new IOException("input principal is invalid."); + } + + if ((keytabFile == null) || (keytabFile.length() <= 0)) { + LOG.error("input keytabFile is invalid."); + throw new IOException("input keytabFile is invalid."); + } + + File userKeytabFile = new File(keytabFile); + if (!userKeytabFile.exists()) { + LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit."); + throw new IOException( + "userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit."); + } + + javax.security.auth.login.Configuration.setConfiguration( + new JaasConfiguration( + loginContextName, principal, userKeytabFile.getAbsolutePath())); + + javax.security.auth.login.Configuration conf = + javax.security.auth.login.Configuration.getConfiguration(); + if (!(conf instanceof JaasConfiguration)) { + LOG.error("javax.security.auth.login.Configuration is not JaasConfiguration."); + throw new IOException( + "javax.security.auth.login.Configuration is not JaasConfiguration."); + } + + AppConfigurationEntry[] entrys = conf.getAppConfigurationEntry(loginContextName); + if (entrys == null) { + LOG.error( + "javax.security.auth.login.Configuration has no AppConfigurationEntry named " + + loginContextName + + "."); + throw new IOException( + "javax.security.auth.login.Configuration has no AppConfigurationEntry named " + + loginContextName + + "."); + } + + boolean checkPrincipal = false; + boolean checkKeytab = true; + for (int i = 0; i < entrys.length; i++) { + if (entrys[i].getOptions().get("principal").equals(principal)) { + checkPrincipal = true; + } + + // if (IS_IBM_JDK) { + // if (entrys[i].getOptions().get("useKeytab").equals(keytabFile)) { + // checkKeytab = true; + // } + // } else { + // if (entrys[i].getOptions().get("keyTab").equals(keytabFile)) { + // checkKeytab = true; + // } + // } + + } + + if (!checkPrincipal) { + LOG.error( + "AppConfigurationEntry named " + + loginContextName + + " does not have principal value of " + + principal + + "."); + throw new IOException( + "AppConfigurationEntry named " + + loginContextName + + " does not have principal value of " + + principal + + "."); + } + + if (!checkKeytab) { + LOG.error( + "AppConfigurationEntry named " + + loginContextName + + " does not have keyTab value of " + + keytabFile + + "."); + throw new IOException( + "AppConfigurationEntry named " + + loginContextName + + " does not have keyTab value of " + + keytabFile + + "."); + } + } + + public static void setZookeeperServerPrincipal(String zkServerPrincipal) throws IOException { + System.setProperty(ZOOKEEPER_SERVER_PRINCIPAL_KEY, zkServerPrincipal); + String ret = System.getProperty(ZOOKEEPER_SERVER_PRINCIPAL_KEY); + if (ret == null) { + LOG.error(ZOOKEEPER_SERVER_PRINCIPAL_KEY + " is null."); + throw new IOException(ZOOKEEPER_SERVER_PRINCIPAL_KEY + " is null."); + } + if (!ret.equals(zkServerPrincipal)) { + LOG.error( + ZOOKEEPER_SERVER_PRINCIPAL_KEY + + " is " + + ret + + " is not " + + zkServerPrincipal + + "."); + throw new IOException( + ZOOKEEPER_SERVER_PRINCIPAL_KEY + + " is " + + ret + + " is not " + + zkServerPrincipal + + "."); + } + } + + public static void setZookeeperServerPrincipal( + String zkServerPrincipalKey, String zkServerPrincipal) throws IOException { + System.setProperty(zkServerPrincipalKey, zkServerPrincipal); + String ret = System.getProperty(zkServerPrincipalKey); + if (ret == null) { + LOG.error(zkServerPrincipalKey + " is null."); + throw new IOException(zkServerPrincipalKey + " is null."); + } + if (!ret.equals(zkServerPrincipal)) { + LOG.error(zkServerPrincipalKey + " is " + ret + " is not " + zkServerPrincipal + "."); + throw new IOException( + zkServerPrincipalKey + " is " + ret + " is not " + zkServerPrincipal + "."); + } + } + + private static void loginHadoop(String principal, String keytabFile) throws IOException { + try { + UserGroupInformation.loginUserFromKeytab(principal, keytabFile); + } catch (IOException e) { + LOG.error("login failed with " + principal + " and " + keytabFile + "."); + LOG.error("perhaps cause 1 is " + LOGIN_FAILED_CAUSE_PASSWORD_WRONG + "."); + LOG.error("perhaps cause 2 is " + LOGIN_FAILED_CAUSE_TIME_WRONG + "."); + LOG.error("perhaps cause 3 is " + LOGIN_FAILED_CAUSE_AES256_WRONG + "."); + LOG.error("perhaps cause 4 is " + LOGIN_FAILED_CAUSE_PRINCIPAL_WRONG + "."); + LOG.error("perhaps cause 5 is " + LOGIN_FAILED_CAUSE_TIME_OUT + "."); + + throw e; + } + } + + private static void checkAuthenticateOverKrb() throws IOException { + UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); + UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); + if (loginUser == null) { + LOG.error("current user is " + currentUser + ", but loginUser is null."); + throw new IOException("current user is " + currentUser + ", but loginUser is null."); + } + if (!loginUser.equals(currentUser)) { + LOG.error("current user is " + currentUser + ", but loginUser is " + loginUser + "."); + throw new IOException( + "current user is " + currentUser + ", but loginUser is " + loginUser + "."); + } + if (!loginUser.hasKerberosCredentials()) { + LOG.error("current user is " + currentUser + " has no Kerberos Credentials."); + throw new IOException( + "current user is " + currentUser + " has no Kerberos Credentials."); + } + if (!UserGroupInformation.isLoginKeytabBased()) { + LOG.error("current user is " + currentUser + " is not Login Keytab Based."); + throw new IOException("current user is " + currentUser + " is not Login Keytab Based."); + } + } + + private static boolean checkCurrentUserCorrect(String principal) throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + if (ugi == null) { + LOG.error("current user still null."); + throw new IOException("current user still null."); + } + + String defaultRealm = null; + try { + defaultRealm = KerberosUtil.getDefaultRealm(); + } catch (Exception e) { + LOG.warn("getDefaultRealm failed."); + throw new IOException(e); + } + + if ((defaultRealm != null) && (defaultRealm.length() > 0)) { + StringBuilder realm = new StringBuilder(); + StringBuilder principalWithRealm = new StringBuilder(); + realm.append("@").append(defaultRealm); + if (!principal.endsWith(realm.toString())) { + principalWithRealm.append(principal).append(realm); + principal = principalWithRealm.toString(); + } + } + + return principal.equals(ugi.getUserName()); + } + + /** + * copy from hbase zkutil 0.94&0.98 A JAAS configuration that defines the login modules that we + * want to use for login. + */ + private static class JaasConfiguration extends javax.security.auth.login.Configuration { + private static final Map BASIC_JAAS_OPTIONS = new HashMap(); + + static { + String jaasEnvVar = System.getenv("HBASE_JAAS_DEBUG"); + if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) { + BASIC_JAAS_OPTIONS.put("debug", "true"); + } + } + + private static final Map KEYTAB_KERBEROS_OPTIONS = + new HashMap(); + + static { + if (IS_IBM_JDK) { + KEYTAB_KERBEROS_OPTIONS.put("credsType", "both"); + } else { + KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true"); + KEYTAB_KERBEROS_OPTIONS.put("useTicketCache", "false"); + KEYTAB_KERBEROS_OPTIONS.put("doNotPrompt", "true"); + KEYTAB_KERBEROS_OPTIONS.put("storeKey", "true"); + } + + KEYTAB_KERBEROS_OPTIONS.putAll(BASIC_JAAS_OPTIONS); + } + + private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN = + new AppConfigurationEntry( + KerberosUtil.getKrb5LoginModuleName(), + LoginModuleControlFlag.REQUIRED, + KEYTAB_KERBEROS_OPTIONS); + + private static final AppConfigurationEntry[] KEYTAB_KERBEROS_CONF = + new AppConfigurationEntry[] {KEYTAB_KERBEROS_LOGIN}; + + private javax.security.auth.login.Configuration baseConfig; + + private final String loginContextName; + + private final boolean useTicketCache; + + private final String keytabFile; + + private final String principal; + + public JaasConfiguration(String loginContextName, String principal, String keytabFile) + throws IOException { + this( + loginContextName, + principal, + keytabFile, + keytabFile == null || keytabFile.length() == 0); + } + + private JaasConfiguration( + String loginContextName, + String principal, + String keytabFile, + boolean useTicketCache) + throws IOException { + try { + this.baseConfig = javax.security.auth.login.Configuration.getConfiguration(); + } catch (SecurityException e) { + this.baseConfig = null; + } + this.loginContextName = loginContextName; + this.useTicketCache = useTicketCache; + this.keytabFile = keytabFile; + this.principal = principal; + + initKerberosOption(); + LOG.info( + "JaasConfiguration loginContextName=" + + loginContextName + + " principal=" + + principal + + " useTicketCache=" + + useTicketCache + + " keytabFile=" + + keytabFile); + } + + private void initKerberosOption() throws IOException { + if (!useTicketCache) { + if (IS_IBM_JDK) { + KEYTAB_KERBEROS_OPTIONS.put("useKeytab", keytabFile); + } else { + KEYTAB_KERBEROS_OPTIONS.put("keyTab", keytabFile); + KEYTAB_KERBEROS_OPTIONS.put("useKeyTab", "true"); + KEYTAB_KERBEROS_OPTIONS.put( + "useTicketCache", useTicketCache ? "true" : "false"); + } + } + KEYTAB_KERBEROS_OPTIONS.put("principal", principal); + } + + public AppConfigurationEntry[] getAppConfigurationEntry(String appName) { + if (loginContextName.equals(appName)) { + return KEYTAB_KERBEROS_CONF; + } + if (baseConfig != null) return baseConfig.getAppConfigurationEntry(appName); + return (null); + } + } +} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/pom.xml b/seatunnel-connectors-v2/connector-kafka-hw/pom.xml deleted file mode 100644 index ebcd81f5ef0..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/pom.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - 4.0.0 - - org.apache.seatunnel - seatunnel-connectors-v2 - ${revision} - - - connector-kafka-hw - SeaTunnel : Connectors V2 : Kafka-hw - - - 3.2.0 - 1.6.4.Final - - - - - - - org.apache.seatunnel - connector-common - ${project.version} - - - org.apache.kafka - kafka-clients - - 2.4.0-hw-ei-315008 - - - org.apache.seatunnel - seatunnel-format-json - ${project.version} - - - org.apache.seatunnel - seatunnel-format-text - ${project.version} - - - org.apache.seatunnel - seatunnel-format-compatible-debezium-json - ${project.version} - - - org.apache.seatunnel - seatunnel-format-compatible-connect-json - ${project.version} - - - org.apache.kafka - connect-json - 1.1.0-mrs-2.1 - - - - - - - true - - - true - - huaweicloudsdk - https://mirrors.huaweicloud.com/repository/maven/huaweicloudsdk/ - - - libs-release - http://192.168.12.88:8082/artifactory/libs-release-local/ - - - libs-snapshot - http://192.168.12.88:8082/artifactory/libs-snapshot-local/ - - - diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/Config.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/Config.java deleted file mode 100644 index 76638e68cd4..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/Config.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.config; - -import org.apache.seatunnel.shade.com.fasterxml.jackson.core.type.TypeReference; - -import org.apache.seatunnel.api.configuration.Option; -import org.apache.seatunnel.api.configuration.Options; - -import java.util.List; -import java.util.Map; - -public class Config { - - public static final String CONNECTOR_IDENTITY = "Kafka"; - /** The default field delimiter is “,” */ - public static final String DEFAULT_FIELD_DELIMITER = ","; - - public static final Option> KAFKA_CONFIG = - Options.key("kafka.config") - .mapType() - .noDefaultValue() - .withDescription( - "In addition to the above parameters that must be specified by the Kafka producer or consumer client, " - + "the user can also specify multiple non-mandatory parameters for the producer or consumer client, " - + "covering all the producer parameters specified in the official Kafka document."); - - public static final Option TOPIC = - Options.key("topic") - .stringType() - .noDefaultValue() - .withDescription( - "Kafka topic name. If there are multiple topics, use , to split, for example: \"tpc1,tpc2\"."); - - public static final Option PATTERN = - Options.key("pattern") - .booleanType() - .defaultValue(false) - .withDescription( - "If pattern is set to true,the regular expression for a pattern of topic names to read from." - + " All topics in clients with names that match the specified regular expression will be subscribed by the consumer."); - - public static final Option BOOTSTRAP_SERVERS = - Options.key("bootstrap.servers") - .stringType() - .noDefaultValue() - .withDescription("Kafka cluster address, separated by \",\"."); - - public static final Option CONSUMER_GROUP = - Options.key("consumer.group") - .stringType() - .defaultValue("SeaTunnel-Consumer-Group") - .withDescription( - "Kafka consumer group id, used to distinguish different consumer groups."); - - public static final Option COMMIT_ON_CHECKPOINT = - Options.key("commit_on_checkpoint") - .booleanType() - .defaultValue(true) - .withDescription( - "If true the consumer's offset will be periodically committed in the background."); - - public static final Option TRANSACTION_PREFIX = - Options.key("transaction_prefix") - .stringType() - .noDefaultValue() - .withDescription( - "If semantic is specified as EXACTLY_ONCE, the producer will write all messages in a Kafka transaction. " - + "Kafka distinguishes different transactions by different transactionId. " - + "This parameter is prefix of kafka transactionId, make sure different job use different prefix."); - - public static final Option SCHEMA = - Options.key("schema") - .objectType(Config.class) - .noDefaultValue() - .withDescription( - "The structure of the data, including field names and field types."); - - public static final Option FORMAT = - Options.key("format") - .enumType(MessageFormat.class) - .defaultValue(MessageFormat.JSON) - .withDescription( - "Data format. The default format is json. Optional text format. The default field separator is \", \". " - + "If you customize the delimiter, add the \"field_delimiter\" option."); - - public static final Option DEBEZIUM_RECORD_INCLUDE_SCHEMA = - Options.key("debezium_record_include_schema") - .booleanType() - .defaultValue(true) - .withDescription("Does the debezium record carry a schema."); - - public static final Option FIELD_DELIMITER = - Options.key("field_delimiter") - .stringType() - .defaultValue(DEFAULT_FIELD_DELIMITER) - .withDescription("Customize the field delimiter for data format."); - - public static final Option PARTITION = - Options.key("partition") - .intType() - .noDefaultValue() - .withDescription( - "We can specify the partition, all messages will be sent to this partition."); - - public static final Option> ASSIGN_PARTITIONS = - Options.key("assign_partitions") - .listType() - .noDefaultValue() - .withDescription( - "We can decide which partition to send based on the content of the message. " - + "The function of this parameter is to distribute information."); - - public static final Option> PARTITION_KEY_FIELDS = - Options.key("partition_key_fields") - .listType() - .noDefaultValue() - .withDescription( - "Configure which fields are used as the key of the kafka message."); - - public static final Option START_MODE = - Options.key("start_mode") - .objectType(StartMode.class) - .defaultValue(StartMode.GROUP_OFFSETS) - .withDescription( - "The initial consumption pattern of consumers,there are several types:\n" - + "[earliest],[group_offsets],[latest],[specific_offsets],[timestamp]"); - - public static final Option START_MODE_TIMESTAMP = - Options.key("start_mode.timestamp") - .longType() - .noDefaultValue() - .withDescription("The time required for consumption mode to be timestamp."); - - public static final Option> START_MODE_OFFSETS = - Options.key("start_mode.offsets") - .type(new TypeReference>() {}) - .noDefaultValue() - .withDescription( - "The offset required for consumption mode to be specific_offsets."); - - /** Configuration key to define the consumer's partition discovery interval, in milliseconds. */ - public static final Option KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS = - Options.key("partition-discovery.interval-millis") - .longType() - .defaultValue(-1L) - .withDescription( - "The interval for dynamically discovering topics and partitions."); - - public static final Option MESSAGE_FORMAT_ERROR_HANDLE_WAY_OPTION = - Options.key("format_error_handle_way") - .enumType(MessageFormatErrorHandleWay.class) - .defaultValue(MessageFormatErrorHandleWay.FAIL) - .withDescription( - "The processing method of data format error. The default value is fail, and the optional value is (fail, skip). " - + "When fail is selected, data format error will block and an exception will be thrown. " - + "When skip is selected, data format error will skip this line data."); - - public static final Option SEMANTICS = - Options.key("semantics") - .enumType(KafkaSemantics.class) - .defaultValue(KafkaSemantics.NON) - .withDescription( - "Semantics that can be chosen EXACTLY_ONCE/AT_LEAST_ONCE/NON, default NON."); -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/KafkaSemantics.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/KafkaSemantics.java deleted file mode 100644 index 5478dc8410f..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/KafkaSemantics.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.config; - -public enum KafkaSemantics { - - /** - * At this semantics, we will directly send the message to kafka, the data may duplicat/lost if - * job restart/retry or network error. - */ - NON, - - /** - * At this semantics, we will retry sending the message to kafka, if the response is not ack. - */ - AT_LEAST_ONCE, - - /** - * AT this semantics, we will use 2pc to guarantee the message is sent to kafka exactly once. - */ - EXACTLY_ONCE, - ; -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/MessageFormat.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/MessageFormat.java deleted file mode 100644 index c756213cf61..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/MessageFormat.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.config; - -public enum MessageFormat { - JSON, - TEXT, - CANAL_JSON, - DEBEZIUM_JSON, - COMPATIBLE_DEBEZIUM_JSON, - COMPATIBLE_KAFKA_CONNECT_JSON, - OGG_JSON -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/MessageFormatErrorHandleWay.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/MessageFormatErrorHandleWay.java deleted file mode 100644 index dff2c4ae7d8..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/MessageFormatErrorHandleWay.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.config; - -public enum MessageFormatErrorHandleWay { - FAIL, - SKIP, -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/StartMode.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/StartMode.java deleted file mode 100644 index dfad20ad934..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/config/StartMode.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.config; - -public enum StartMode { - EARLIEST("earliest"), - - GROUP_OFFSETS("group_offsets"), - - LATEST("latest"), - - TIMESTAMP("timestamp"), - - SPECIFIC_OFFSETS("specific_offsets"); - - private String mode; - - StartMode(String mode) { - this.mode = mode; - } - - public String getMode() { - return mode; - } - - @Override - public String toString() { - return mode; - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/exception/KafkaConnectorErrorCode.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/exception/KafkaConnectorErrorCode.java deleted file mode 100644 index b4f243239db..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/exception/KafkaConnectorErrorCode.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception; - -import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; - -public enum KafkaConnectorErrorCode implements SeaTunnelErrorCode { - VERSION_INCOMPATIBLE("KAFKA-01", "Incompatible KafkaProducer version"), - GET_TRANSACTIONMANAGER_FAILED("KAFKA-02", "Get transactionManager in KafkaProducer failed"), - ADD_SPLIT_CHECKPOINT_FAILED("KAFKA-03", "Add the split checkpoint state to reader failed"), - ADD_SPLIT_BACK_TO_ENUMERATOR_FAILED( - "KAFKA-04", - "Add a split back to the split enumerator failed,it will only happen when a SourceReader failed"), - CONSUME_THREAD_RUN_ERROR( - "KAFKA-05", "Error occurred when the kafka consumer thread was running"), - CONSUME_DATA_FAILED("KAFKA-06", "Kafka failed to consume data"), - CONSUMER_CLOSE_FAILED("KAFKA-07", "Kafka failed to close consumer"); - - private final String code; - private final String description; - - KafkaConnectorErrorCode(String code, String description) { - this.code = code; - this.description = description; - } - - @Override - public String getCode() { - return this.code; - } - - @Override - public String getDescription() { - return this.description; - } - - @Override - public String getErrorMessage() { - return SeaTunnelErrorCode.super.getErrorMessage(); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/exception/KafkaConnectorException.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/exception/KafkaConnectorException.java deleted file mode 100644 index 5064fd1953d..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/exception/KafkaConnectorException.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception; - -import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; -import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; - -public class KafkaConnectorException extends SeaTunnelRuntimeException { - public KafkaConnectorException(SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage) { - super(seaTunnelErrorCode, errorMessage); - } - - public KafkaConnectorException( - SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage, Throwable cause) { - super(seaTunnelErrorCode, errorMessage, cause); - } - - public KafkaConnectorException(SeaTunnelErrorCode seaTunnelErrorCode, Throwable cause) { - super(seaTunnelErrorCode, cause); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/serialize/DefaultSeaTunnelRowSerializer.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/serialize/DefaultSeaTunnelRowSerializer.java deleted file mode 100644 index da3d9b347db..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/serialize/DefaultSeaTunnelRowSerializer.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.serialize; - -import org.apache.seatunnel.api.serialization.SerializationSchema; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.MessageFormat; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorException; -import org.apache.seatunnel.format.compatible.debezium.json.CompatibleDebeziumJsonDeserializationSchema; -import org.apache.seatunnel.format.compatible.debezium.json.CompatibleDebeziumJsonSerializationSchema; -import org.apache.seatunnel.format.json.JsonSerializationSchema; -import org.apache.seatunnel.format.json.canal.CanalJsonSerializationSchema; -import org.apache.seatunnel.format.json.debezium.DebeziumJsonSerializationSchema; -import org.apache.seatunnel.format.json.exception.SeaTunnelJsonFormatException; -import org.apache.seatunnel.format.json.ogg.OggJsonSerializationSchema; -import org.apache.seatunnel.format.text.TextSerializationSchema; - -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.header.Header; - -import lombok.RequiredArgsConstructor; - -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@RequiredArgsConstructor -public class DefaultSeaTunnelRowSerializer implements SeaTunnelRowSerializer { - private final Function topicExtractor; - private final Function partitionExtractor; - private final Function timestampExtractor; - private final Function keyExtractor; - private final Function valueExtractor; - private final Function> headersExtractor; - - @Override - public ProducerRecord serializeRow(SeaTunnelRow row) { - return new ProducerRecord( - topicExtractor.apply(row), - partitionExtractor.apply(row), - timestampExtractor.apply(row), - keyExtractor.apply(row), - valueExtractor.apply(row), - headersExtractor.apply(row)); - } - - public static DefaultSeaTunnelRowSerializer create( - String topic, SeaTunnelRowType rowType, MessageFormat format, String delimiter) { - return new DefaultSeaTunnelRowSerializer( - topicExtractor(topic, rowType, format), - partitionExtractor(null), - timestampExtractor(), - keyExtractor(null, rowType, format, delimiter), - valueExtractor(rowType, format, delimiter), - headersExtractor()); - } - - public static DefaultSeaTunnelRowSerializer create( - String topic, - Integer partition, - SeaTunnelRowType rowType, - MessageFormat format, - String delimiter) { - return new DefaultSeaTunnelRowSerializer( - topicExtractor(topic, rowType, format), - partitionExtractor(partition), - timestampExtractor(), - keyExtractor(null, rowType, format, delimiter), - valueExtractor(rowType, format, delimiter), - headersExtractor()); - } - - public static DefaultSeaTunnelRowSerializer create( - String topic, - List keyFields, - SeaTunnelRowType rowType, - MessageFormat format, - String delimiter) { - return new DefaultSeaTunnelRowSerializer( - topicExtractor(topic, rowType, format), - partitionExtractor(null), - timestampExtractor(), - keyExtractor(keyFields, rowType, format, delimiter), - valueExtractor(rowType, format, delimiter), - headersExtractor()); - } - - private static Function partitionExtractor(Integer partition) { - return row -> partition; - } - - private static Function timestampExtractor() { - return row -> null; - } - - private static Function> headersExtractor() { - return row -> null; - } - - private static Function topicExtractor( - String topic, SeaTunnelRowType rowType, MessageFormat format) { - if (MessageFormat.COMPATIBLE_DEBEZIUM_JSON.equals(format)) { - int topicFieldIndex = - rowType.indexOf(CompatibleDebeziumJsonDeserializationSchema.FIELD_TOPIC); - return row -> row.getField(topicFieldIndex).toString(); - } - - String regex = "\\$\\{(.*?)\\}"; - Pattern pattern = Pattern.compile(regex, Pattern.DOTALL); - Matcher matcher = pattern.matcher(topic); - boolean isExtractTopic = matcher.find(); - if (!isExtractTopic) { - return row -> topic; - } - - String topicField = matcher.group(1); - List fieldNames = Arrays.asList(rowType.getFieldNames()); - if (!fieldNames.contains(topicField)) { - throw new KafkaConnectorException( - CommonErrorCodeDeprecated.ILLEGAL_ARGUMENT, - String.format("Field name { %s } is not found!", topic)); - } - int topicFieldIndex = rowType.indexOf(topicField); - return row -> { - Object topicFieldValue = row.getField(topicFieldIndex); - if (topicFieldValue == null) { - throw new KafkaConnectorException( - CommonErrorCodeDeprecated.ILLEGAL_ARGUMENT, "The column value is empty!"); - } - return topicFieldValue.toString(); - }; - } - - private static Function keyExtractor( - List keyFields, - SeaTunnelRowType rowType, - MessageFormat format, - String delimiter) { - if (MessageFormat.COMPATIBLE_DEBEZIUM_JSON.equals(format)) { - CompatibleDebeziumJsonSerializationSchema serializationSchema = - new CompatibleDebeziumJsonSerializationSchema(rowType, true); - return row -> serializationSchema.serialize(row); - } - - if (keyFields == null || keyFields.isEmpty()) { - return row -> null; - } - - SeaTunnelRowType keyType = createKeyType(keyFields, rowType); - Function keyRowExtractor = - createKeyRowExtractor(keyType, rowType); - SerializationSchema serializationSchema = - createSerializationSchema(keyType, format, delimiter, true); - return row -> serializationSchema.serialize(keyRowExtractor.apply(row)); - } - - private static Function valueExtractor( - SeaTunnelRowType rowType, MessageFormat format, String delimiter) { - SerializationSchema serializationSchema = - createSerializationSchema(rowType, format, delimiter, false); - return row -> serializationSchema.serialize(row); - } - - private static SeaTunnelRowType createKeyType( - List keyFieldNames, SeaTunnelRowType rowType) { - int[] keyFieldIndexArr = new int[keyFieldNames.size()]; - SeaTunnelDataType[] keyFieldDataTypeArr = new SeaTunnelDataType[keyFieldNames.size()]; - for (int i = 0; i < keyFieldNames.size(); i++) { - String keyFieldName = keyFieldNames.get(i); - int rowFieldIndex = rowType.indexOf(keyFieldName); - keyFieldIndexArr[i] = rowFieldIndex; - keyFieldDataTypeArr[i] = rowType.getFieldType(rowFieldIndex); - } - return new SeaTunnelRowType(keyFieldNames.toArray(new String[0]), keyFieldDataTypeArr); - } - - private static Function createKeyRowExtractor( - SeaTunnelRowType keyType, SeaTunnelRowType rowType) { - int[] keyIndex = new int[keyType.getTotalFields()]; - for (int i = 0; i < keyType.getTotalFields(); i++) { - keyIndex[i] = rowType.indexOf(keyType.getFieldName(i)); - } - return row -> { - Object[] fields = new Object[keyType.getTotalFields()]; - for (int i = 0; i < keyIndex.length; i++) { - fields[i] = row.getField(keyIndex[i]); - } - return new SeaTunnelRow(fields); - }; - } - - private static SerializationSchema createSerializationSchema( - SeaTunnelRowType rowType, MessageFormat format, String delimiter, boolean isKey) { - switch (format) { - case JSON: - return new JsonSerializationSchema(rowType); - case TEXT: - return TextSerializationSchema.builder() - .seaTunnelRowType(rowType) - .delimiter(delimiter) - .build(); - case CANAL_JSON: - return new CanalJsonSerializationSchema(rowType); - case OGG_JSON: - return new OggJsonSerializationSchema(rowType); - case DEBEZIUM_JSON: - return new DebeziumJsonSerializationSchema(rowType); - case COMPATIBLE_DEBEZIUM_JSON: - return new CompatibleDebeziumJsonSerializationSchema(rowType, isKey); - default: - throw new SeaTunnelJsonFormatException( - CommonErrorCodeDeprecated.UNSUPPORTED_DATA_TYPE, - "Unsupported format: " + format); - } - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/serialize/SeaTunnelRowSerializer.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/serialize/SeaTunnelRowSerializer.java deleted file mode 100644 index 8a4191ad338..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/serialize/SeaTunnelRowSerializer.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.serialize; - -import org.apache.seatunnel.api.table.type.SeaTunnelRow; - -import org.apache.kafka.clients.producer.ProducerRecord; - -public interface SeaTunnelRowSerializer { - - /** - * Serialize the {@link SeaTunnelRow} to a Kafka {@link ProducerRecord}. - * - * @param row seatunnel row - * @return kafka record. - */ - ProducerRecord serializeRow(SeaTunnelRow row); -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaInternalProducer.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaInternalProducer.java deleted file mode 100644 index 4edfa8d0b10..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaInternalProducer.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.seatunnel.common.utils.ReflectionUtils; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorException; - -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.internals.TransactionManager; -import org.apache.kafka.common.errors.ProducerFencedException; - -import lombok.extern.slf4j.Slf4j; - -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.util.Optional; -import java.util.Properties; - -/** A {@link KafkaProducer} that allow resume transaction from transactionId */ -@Slf4j -public class KafkaInternalProducer extends KafkaProducer { - - private static final String TRANSACTION_MANAGER_STATE_ENUM = - "org.apache.kafka.clients.producer.internals.TransactionManager$State"; - private static final String PRODUCER_ID_AND_EPOCH_FIELD_NAME = "producerIdAndEpoch"; - private String transactionalId; - - public KafkaInternalProducer(Properties properties, String transactionId) { - super(properties); - this.transactionalId = transactionId; - } - - @Override - public void initTransactions() { - setTransactionalId(this.transactionalId); - super.initTransactions(); - } - - @Override - public void beginTransaction() throws ProducerFencedException { - super.beginTransaction(); - } - - @Override - public void commitTransaction() throws ProducerFencedException { - super.commitTransaction(); - } - - @Override - public void abortTransaction() throws ProducerFencedException { - super.abortTransaction(); - } - - public void setTransactionalId(String transactionalId) { - if (!transactionalId.equals(this.transactionalId)) { - Object transactionManager = getTransactionManager(); - synchronized (transactionManager) { - ReflectionUtils.setField(transactionManager, "transactionalId", transactionalId); - ReflectionUtils.setField( - transactionManager, - "currentState", - getTransactionManagerState("UNINITIALIZED")); - this.transactionalId = transactionalId; - } - } - } - - public short getEpoch() { - Object transactionManager = getTransactionManager(); - Optional producerIdAndEpoch = - ReflectionUtils.getField(transactionManager, PRODUCER_ID_AND_EPOCH_FIELD_NAME); - return (short) ReflectionUtils.getField(producerIdAndEpoch.get(), "epoch").get(); - } - - public long getProducerId() { - Object transactionManager = getTransactionManager(); - Object producerIdAndEpoch = - ReflectionUtils.getField(transactionManager, PRODUCER_ID_AND_EPOCH_FIELD_NAME) - .get(); - return (long) ReflectionUtils.getField(producerIdAndEpoch, "producerId").get(); - } - - public void resumeTransaction(long producerId, short epoch) { - - log.info( - "Attempting to resume transaction {} with producerId {} and epoch {}", - transactionalId, - producerId, - epoch); - - Object transactionManager = getTransactionManager(); - synchronized (transactionManager) { - Object topicPartitionBookkeeper = - ReflectionUtils.getField( - transactionManager, - transactionManager.getClass(), - "topicPartitionBookkeeper") - .get(); - - transitionTransactionManagerStateTo(transactionManager, "INITIALIZING"); - ReflectionUtils.invoke(topicPartitionBookkeeper, "reset"); - - ReflectionUtils.setField( - transactionManager, - PRODUCER_ID_AND_EPOCH_FIELD_NAME, - createProducerIdAndEpoch(producerId, epoch)); - - transitionTransactionManagerStateTo(transactionManager, "READY"); - - transitionTransactionManagerStateTo(transactionManager, "IN_TRANSACTION"); - ReflectionUtils.setField(transactionManager, "transactionStarted", true); - } - } - - private static Object createProducerIdAndEpoch(long producerId, short epoch) { - try { - Field field = - TransactionManager.class.getDeclaredField(PRODUCER_ID_AND_EPOCH_FIELD_NAME); - Class clazz = field.getType(); - Constructor constructor = clazz.getDeclaredConstructor(Long.TYPE, Short.TYPE); - constructor.setAccessible(true); - return constructor.newInstance(producerId, epoch); - } catch (InvocationTargetException - | InstantiationException - | IllegalAccessException - | NoSuchFieldException - | NoSuchMethodException e) { - throw new KafkaConnectorException( - KafkaConnectorErrorCode.VERSION_INCOMPATIBLE, - "Incompatible KafkaProducer version", - e); - } - } - - private Object getTransactionManager() { - Optional transactionManagerOptional = - ReflectionUtils.getField(this, KafkaProducer.class, "transactionManager"); - if (!transactionManagerOptional.isPresent()) { - throw new KafkaConnectorException( - KafkaConnectorErrorCode.GET_TRANSACTIONMANAGER_FAILED, - "Can't get transactionManager in KafkaProducer"); - } - return transactionManagerOptional.get(); - } - - private static void transitionTransactionManagerStateTo( - Object transactionManager, String state) { - ReflectionUtils.invoke( - transactionManager, "transitionTo", getTransactionManagerState(state)); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private static Enum getTransactionManagerState(String enumName) { - try { - Class cl = (Class) Class.forName(TRANSACTION_MANAGER_STATE_ENUM); - return Enum.valueOf(cl, enumName); - } catch (ClassNotFoundException e) { - throw new KafkaConnectorException( - KafkaConnectorErrorCode.VERSION_INCOMPATIBLE, - "Incompatible KafkaProducer version", - e); - } - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaNoTransactionSender.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaNoTransactionSender.java deleted file mode 100644 index d1f9deb7748..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaNoTransactionSender.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaSinkState; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.utils.LoginUtil; - -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Properties; - -/** - * This sender will send the data to the Kafka, and will not guarantee the data is committed to the - * Kafka exactly-once. - * - * @param key type. - * @param value type. - */ -public class KafkaNoTransactionSender implements KafkaProduceSender { - - private KafkaProducer kafkaProducer; - - /** 用户自己申请的机机账号keytab文件名称 */ - private static final String USER_KEYTAB_FILE = "user.keytab"; - - /** 用户自己申请的机机账号名称 */ - private static final String USER_PRINCIPAL = "venuskafka"; - - public KafkaNoTransactionSender(Properties properties) { - this.kafkaProducer = new KafkaProducer<>(properties); - } - - public void securityPrepare() throws IOException { - String filePath = - "D:\\code\\seatunnel-dev\\incubator-seatunnel\\seatunnel-connectors-v2\\connector-kafka-hw\\src\\main\\resources\\conf\\"; - String krbFile = filePath + "krb5.conf"; - String userKeyTableFile = filePath + USER_KEYTAB_FILE; - - // windows路径下分隔符替换 - userKeyTableFile = userKeyTableFile.replace("\\", "\\\\"); - krbFile = krbFile.replace("\\", "\\\\"); - - LoginUtil.setKrb5Config(krbFile); - LoginUtil.setZookeeperServerPrincipal("zookeeper/hadoop.hadoop.com"); - LoginUtil.setJaasFile(USER_PRINCIPAL, userKeyTableFile); - } - - @Override - public void send(ProducerRecord producerRecord) { - kafkaProducer.send(producerRecord); - } - - @Override - public void beginTransaction(String transactionId) { - // no-op - } - - @Override - public Optional prepareCommit() { - return Optional.empty(); - } - - @Override - public void abortTransaction() { - // no-op - } - - @Override - public void abortTransaction(long checkpointId) { - // no-op - } - - @Override - public List snapshotState(long checkpointId) { - kafkaProducer.flush(); - return Collections.emptyList(); - } - - @Override - public void close() { - kafkaProducer.flush(); - kafkaProducer.close(); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaProduceSender.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaProduceSender.java deleted file mode 100644 index bdb0e489007..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaProduceSender.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaSinkState; - -import org.apache.kafka.clients.producer.ProducerRecord; - -import java.util.List; -import java.util.Optional; - -public interface KafkaProduceSender extends AutoCloseable { - /** - * Send data to kafka. - * - * @param producerRecord data to send - */ - void send(ProducerRecord producerRecord); - - void beginTransaction(String transactionId); - - /** - * Prepare a transaction commit. - * - * @return commit info, or empty if no commit is needed. - */ - Optional prepareCommit(); - - /** Abort the current transaction. */ - void abortTransaction(); - - /** - * Abort the given transaction. - * - * @param checkpointId the id of the last checkpoint of the last run - */ - void abortTransaction(long checkpointId); - - /** - * Get the current kafka state of the sender. - * - * @return kafka state List, or empty if no state is available. - */ - List snapshotState(long checkpointId); -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSink.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSink.java deleted file mode 100644 index e624c0cdc28..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSink.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.serialization.DefaultSerializer; -import org.apache.seatunnel.api.serialization.Serializer; -import org.apache.seatunnel.api.sink.SeaTunnelSink; -import org.apache.seatunnel.api.sink.SinkCommitter; -import org.apache.seatunnel.api.sink.SinkWriter; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaAggregatedCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaSinkState; - -import java.util.Collections; -import java.util.List; -import java.util.Optional; - -/** - * Kafka Sink implementation by using SeaTunnel sink API. This class contains the method to create - * {@link KafkaSinkWriter} and {@link KafkaSinkCommitter}. - */ -public class KafkaSink - implements SeaTunnelSink< - SeaTunnelRow, KafkaSinkState, KafkaCommitInfo, KafkaAggregatedCommitInfo> { - - private final ReadonlyConfig pluginConfig; - private final SeaTunnelRowType seaTunnelRowType; - - public KafkaSink(ReadonlyConfig pluginConfig, SeaTunnelRowType rowType) { - this.pluginConfig = pluginConfig; - this.seaTunnelRowType = rowType; - } - - @Override - public SinkWriter createWriter( - SinkWriter.Context context) { - return new KafkaSinkWriter( - context, seaTunnelRowType, pluginConfig, Collections.emptyList()); - } - - @Override - public SinkWriter restoreWriter( - SinkWriter.Context context, List states) { - return new KafkaSinkWriter(context, seaTunnelRowType, pluginConfig, states); - } - - @Override - public Optional> getWriterStateSerializer() { - return Optional.of(new DefaultSerializer<>()); - } - - @Override - public Optional> createCommitter() { - return Optional.of(new KafkaSinkCommitter(pluginConfig)); - } - - @Override - public Optional> getCommitInfoSerializer() { - return Optional.of(new DefaultSerializer<>()); - } - - @Override - public String getPluginName() { - return org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.CONNECTOR_IDENTITY; - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkCommitter.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkCommitter.java deleted file mode 100644 index 6548d6225dc..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkCommitter.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.sink.SinkCommitter; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaCommitInfo; - -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; - -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.Properties; - -@Slf4j -public class KafkaSinkCommitter implements SinkCommitter { - - private final ReadonlyConfig pluginConfig; - - private KafkaInternalProducer kafkaProducer; - - public KafkaSinkCommitter(ReadonlyConfig pluginConfig) { - this.pluginConfig = pluginConfig; - } - - @Override - public List commit(List commitInfos) { - if (commitInfos.isEmpty()) { - return commitInfos; - } - for (KafkaCommitInfo commitInfo : commitInfos) { - String transactionId = commitInfo.getTransactionId(); - if (log.isDebugEnabled()) { - log.debug("Committing transaction {}", transactionId); - } - KafkaProducer producer = getProducer(commitInfo); - producer.commitTransaction(); - producer.flush(); - } - if (this.kafkaProducer != null) { - kafkaProducer.close(); - kafkaProducer = null; - } - return commitInfos; - } - - @Override - public void abort(List commitInfos) { - if (commitInfos.isEmpty()) { - return; - } - for (KafkaCommitInfo commitInfo : commitInfos) { - KafkaProducer producer = getProducer(commitInfo); - producer.abortTransaction(); - } - if (this.kafkaProducer != null) { - kafkaProducer.close(); - kafkaProducer = null; - } - } - - private KafkaInternalProducer getProducer(KafkaCommitInfo commitInfo) { - if (this.kafkaProducer != null) { - this.kafkaProducer.setTransactionalId(commitInfo.getTransactionId()); - } else { - Properties kafkaProperties = commitInfo.getKafkaProperties(); - kafkaProperties.setProperty( - ProducerConfig.TRANSACTIONAL_ID_CONFIG, commitInfo.getTransactionId()); - kafkaProducer = - new KafkaInternalProducer<>( - commitInfo.getKafkaProperties(), commitInfo.getTransactionId()); - } - kafkaProducer.resumeTransaction(commitInfo.getProducerId(), commitInfo.getEpoch()); - return kafkaProducer; - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkFactory.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkFactory.java deleted file mode 100644 index 4c0af8da4ad..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkFactory.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.table.connector.TableSink; -import org.apache.seatunnel.api.table.factory.Factory; -import org.apache.seatunnel.api.table.factory.TableSinkFactory; -import org.apache.seatunnel.api.table.factory.TableSinkFactoryContext; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.MessageFormat; - -import com.google.auto.service.AutoService; - -import java.util.Arrays; - -@AutoService(Factory.class) -public class KafkaSinkFactory implements TableSinkFactory { - @Override - public String factoryIdentifier() { - return "Kafka"; - } - - @Override - public OptionRule optionRule() { - return OptionRule.builder() - .required(Config.FORMAT, Config.BOOTSTRAP_SERVERS) - .conditional( - Config.FORMAT, - Arrays.asList( - MessageFormat.JSON, - MessageFormat.CANAL_JSON, - MessageFormat.TEXT, - MessageFormat.OGG_JSON), - Config.TOPIC) - .optional( - Config.KAFKA_CONFIG, - Config.ASSIGN_PARTITIONS, - Config.TRANSACTION_PREFIX, - Config.SEMANTICS, - Config.PARTITION, - Config.PARTITION_KEY_FIELDS) - .build(); - } - - @Override - public TableSink createSink(TableSinkFactoryContext context) { - return () -> - new KafkaSink( - context.getOptions(), - context.getCatalogTable().getTableSchema().toPhysicalRowDataType()); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkWriter.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkWriter.java deleted file mode 100644 index de59920104b..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaSinkWriter.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.sink.SinkWriter; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.KafkaSemantics; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.MessageFormat; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorException; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.serialize.DefaultSeaTunnelRowSerializer; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.serialize.SeaTunnelRowSerializer; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaSinkState; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.ByteArraySerializer; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Properties; -import java.util.Random; - -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.ASSIGN_PARTITIONS; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.BOOTSTRAP_SERVERS; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.DEFAULT_FIELD_DELIMITER; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.FIELD_DELIMITER; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.FORMAT; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.KAFKA_CONFIG; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.PARTITION; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.PARTITION_KEY_FIELDS; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.SEMANTICS; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.TOPIC; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.TRANSACTION_PREFIX; - -/** KafkaSinkWriter is a sink writer that will write {@link SeaTunnelRow} to Kafka. */ -public class KafkaSinkWriter implements SinkWriter { - - private final SinkWriter.Context context; - - private String transactionPrefix; - private long lastCheckpointId = 0; - private SeaTunnelRowType seaTunnelRowType; - - private final KafkaProduceSender kafkaProducerSender; - private final SeaTunnelRowSerializer seaTunnelRowSerializer; - - private static final int PREFIX_RANGE = 10000; - - public KafkaSinkWriter( - SinkWriter.Context context, - SeaTunnelRowType seaTunnelRowType, - ReadonlyConfig pluginConfig, - List kafkaStates) { - this.context = context; - this.seaTunnelRowType = seaTunnelRowType; - if (pluginConfig.get(ASSIGN_PARTITIONS) != null - && !CollectionUtils.isEmpty(pluginConfig.get(ASSIGN_PARTITIONS))) { - MessageContentPartitioner.setAssignPartitions(pluginConfig.get(ASSIGN_PARTITIONS)); - } - - if (pluginConfig.get(TRANSACTION_PREFIX) != null) { - this.transactionPrefix = pluginConfig.get(TRANSACTION_PREFIX); - } else { - Random random = new Random(); - this.transactionPrefix = String.format("SeaTunnel%04d", random.nextInt(PREFIX_RANGE)); - } - - restoreState(kafkaStates); - this.seaTunnelRowSerializer = getSerializer(pluginConfig, seaTunnelRowType); - if (KafkaSemantics.EXACTLY_ONCE.equals(getKafkaSemantics(pluginConfig))) { - this.kafkaProducerSender = - new KafkaTransactionSender<>( - this.transactionPrefix, getKafkaProperties(pluginConfig)); - // abort all transaction number bigger than current transaction, because they maybe - // already start - // transaction. - if (!kafkaStates.isEmpty()) { - this.kafkaProducerSender.abortTransaction(kafkaStates.get(0).getCheckpointId() + 1); - } - this.kafkaProducerSender.beginTransaction( - generateTransactionId(this.transactionPrefix, this.lastCheckpointId + 1)); - } else { - this.kafkaProducerSender = - new KafkaNoTransactionSender<>(getKafkaProperties(pluginConfig)); - } - } - - @Override - public void write(SeaTunnelRow element) { - ProducerRecord producerRecord = - seaTunnelRowSerializer.serializeRow(element); - kafkaProducerSender.send(producerRecord); - } - - @Override - public List snapshotState(long checkpointId) { - List states = kafkaProducerSender.snapshotState(checkpointId); - this.lastCheckpointId = checkpointId; - this.kafkaProducerSender.beginTransaction( - generateTransactionId(this.transactionPrefix, this.lastCheckpointId + 1)); - return states; - } - - @Override - public Optional prepareCommit() { - return kafkaProducerSender.prepareCommit(); - } - - @Override - public void abortPrepare() { - kafkaProducerSender.abortTransaction(); - } - - @Override - public void close() { - try { - kafkaProducerSender.close(); - } catch (Exception e) { - throw new KafkaConnectorException( - CommonErrorCodeDeprecated.WRITER_OPERATION_FAILED, - "Close kafka sink writer error", - e); - } - } - - private Properties getKafkaProperties(ReadonlyConfig pluginConfig) { - Properties kafkaProperties = new Properties(); - if (pluginConfig.get(KAFKA_CONFIG) != null) { - pluginConfig.get(KAFKA_CONFIG).forEach((key, value) -> kafkaProperties.put(key, value)); - } - - if (pluginConfig.get(ASSIGN_PARTITIONS) != null) { - kafkaProperties.put( - ProducerConfig.PARTITIONER_CLASS_CONFIG, - "org.apache.seatunnel.connectors.seatunnel.kafka.sink.MessageContentPartitioner"); - } - - kafkaProperties.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, pluginConfig.get(BOOTSTRAP_SERVERS)); - kafkaProperties.put( - ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); - kafkaProperties.put( - ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); - return kafkaProperties; - } - - private SeaTunnelRowSerializer getSerializer( - ReadonlyConfig pluginConfig, SeaTunnelRowType seaTunnelRowType) { - MessageFormat messageFormat = pluginConfig.get(FORMAT); - String delimiter = DEFAULT_FIELD_DELIMITER; - - if (pluginConfig.get(FIELD_DELIMITER) != null) { - delimiter = pluginConfig.get(FIELD_DELIMITER); - } - - String topic = pluginConfig.get(TOPIC); - if (pluginConfig.get(PARTITION_KEY_FIELDS) != null && pluginConfig.get(PARTITION) != null) { - throw new KafkaConnectorException( - KafkaConnectorErrorCode.GET_TRANSACTIONMANAGER_FAILED, - "Cannot select both `partiton` and `partition_key_fields`. You can configure only one of them"); - } - if (pluginConfig.get(PARTITION_KEY_FIELDS) != null) { - return DefaultSeaTunnelRowSerializer.create( - topic, - getPartitionKeyFields(pluginConfig, seaTunnelRowType), - seaTunnelRowType, - messageFormat, - delimiter); - } - if (pluginConfig.get(PARTITION) != null) { - return DefaultSeaTunnelRowSerializer.create( - topic, pluginConfig.get(PARTITION), seaTunnelRowType, messageFormat, delimiter); - } - // By default, all partitions are sent randomly - return DefaultSeaTunnelRowSerializer.create( - topic, Arrays.asList(), seaTunnelRowType, messageFormat, delimiter); - } - - private KafkaSemantics getKafkaSemantics(ReadonlyConfig pluginConfig) { - if (pluginConfig.get(SEMANTICS) != null) { - return pluginConfig.get(SEMANTICS); - } - return KafkaSemantics.NON; - } - - protected static String generateTransactionId(String transactionPrefix, long checkpointId) { - return transactionPrefix + "-" + checkpointId; - } - - private void restoreState(List states) { - if (!states.isEmpty()) { - this.transactionPrefix = states.get(0).getTransactionIdPrefix(); - this.lastCheckpointId = states.get(0).getCheckpointId(); - } - } - - private List getPartitionKeyFields( - ReadonlyConfig pluginConfig, SeaTunnelRowType seaTunnelRowType) { - - if (pluginConfig.get(PARTITION_KEY_FIELDS) != null) { - List partitionKeyFields = pluginConfig.get(PARTITION_KEY_FIELDS); - List rowTypeFieldNames = Arrays.asList(seaTunnelRowType.getFieldNames()); - for (String partitionKeyField : partitionKeyFields) { - if (!rowTypeFieldNames.contains(partitionKeyField)) { - throw new KafkaConnectorException( - CommonErrorCodeDeprecated.ILLEGAL_ARGUMENT, - String.format( - "Partition key field not found: %s, rowType: %s", - partitionKeyField, rowTypeFieldNames)); - } - } - return partitionKeyFields; - } - return Collections.emptyList(); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaTransactionSender.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaTransactionSender.java deleted file mode 100644 index c4be74f89ef..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/KafkaTransactionSender.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaCommitInfo; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaSinkState; - -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.Optional; -import java.util.Properties; - -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink.KafkaSinkWriter.generateTransactionId; - -/** - * This sender will use kafka transaction to guarantee the data is sent to kafka at exactly-once. - * - * @param key type. - * @param value type. - */ -@Slf4j -public class KafkaTransactionSender implements KafkaProduceSender { - - private KafkaInternalProducer kafkaProducer; - private String transactionId; - private final String transactionPrefix; - private final Properties kafkaProperties; - - public KafkaTransactionSender(String transactionPrefix, Properties kafkaProperties) { - this.transactionPrefix = transactionPrefix; - this.kafkaProperties = kafkaProperties; - } - - @Override - public void send(ProducerRecord producerRecord) { - kafkaProducer.send(producerRecord); - } - - @Override - public void beginTransaction(String transactionId) { - this.transactionId = transactionId; - this.kafkaProducer = getTransactionProducer(kafkaProperties, transactionId); - kafkaProducer.beginTransaction(); - } - - @Override - public Optional prepareCommit() { - KafkaCommitInfo kafkaCommitInfo = - new KafkaCommitInfo( - transactionId, - kafkaProperties, - this.kafkaProducer.getProducerId(), - this.kafkaProducer.getEpoch()); - return Optional.of(kafkaCommitInfo); - } - - @Override - public void abortTransaction() { - kafkaProducer.abortTransaction(); - } - - @Override - public void abortTransaction(long checkpointId) { - - KafkaInternalProducer producer; - if (this.kafkaProducer != null) { - producer = this.kafkaProducer; - } else { - producer = - getTransactionProducer( - this.kafkaProperties, - generateTransactionId(this.transactionPrefix, checkpointId)); - } - - for (long i = checkpointId; ; i++) { - String transactionId = generateTransactionId(this.transactionPrefix, i); - producer.setTransactionalId(transactionId); - if (log.isDebugEnabled()) { - log.debug("Abort kafka transaction: {}", transactionId); - } - producer.flush(); - if (producer.getEpoch() == 0) { - break; - } - } - } - - @Override - public List snapshotState(long checkpointId) { - return Lists.newArrayList( - new KafkaSinkState( - transactionId, transactionPrefix, checkpointId, kafkaProperties)); - } - - @Override - public void close() { - if (kafkaProducer != null) { - kafkaProducer.flush(); - kafkaProducer.close(); - } - } - - private KafkaInternalProducer getTransactionProducer( - Properties properties, String transactionId) { - close(); - Properties transactionProperties = (Properties) properties.clone(); - transactionProperties.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, transactionId); - KafkaInternalProducer transactionProducer = - new KafkaInternalProducer<>(transactionProperties, transactionId); - transactionProducer.initTransactions(); - return transactionProducer; - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/MessageContentPartitioner.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/MessageContentPartitioner.java deleted file mode 100644 index e2b55328847..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/sink/MessageContentPartitioner.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink; - -import org.apache.kafka.clients.producer.Partitioner; -import org.apache.kafka.common.Cluster; -import org.apache.kafka.common.PartitionInfo; - -import java.util.List; -import java.util.Map; - -public class MessageContentPartitioner implements Partitioner { - private static List ASSIGNPARTITIONS; - - public static void setAssignPartitions(List assignPartitionList) { - ASSIGNPARTITIONS = assignPartitionList; - } - - @Override - public int partition( - String topic, - Object key, - byte[] keyBytes, - Object value, - byte[] valueBytes, - Cluster cluster) { - List partitions = cluster.partitionsForTopic(topic); - int numPartitions = partitions.size(); - - int assignPartitionsSize = ASSIGNPARTITIONS.size(); - String message = new String(valueBytes); - for (int i = 0; i < assignPartitionsSize; i++) { - if (message.contains(ASSIGNPARTITIONS.get(i))) { - return i; - } - } - // Choose one of the remaining partitions according to the hashcode. - return ((message.hashCode() & Integer.MAX_VALUE) % (numPartitions - assignPartitionsSize)) - + assignPartitionsSize; - } - - @Override - public void close() {} - - @Override - public void configure(Map map) {} -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/ConsumerMetadata.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/ConsumerMetadata.java deleted file mode 100644 index 4c3f033c762..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/ConsumerMetadata.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.source; - -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.StartMode; - -import org.apache.kafka.common.TopicPartition; - -import lombok.Data; - -import java.io.Serializable; -import java.util.Map; -import java.util.Properties; - -/** Kafka consumer metadata, include topic, bootstrap server etc. */ -@Data -public class ConsumerMetadata implements Serializable { - - private String topic; - private boolean isPattern = false; - private String bootstrapServers; - private Properties properties; - private String consumerGroup; - private boolean commitOnCheckpoint = false; - private StartMode startMode = StartMode.GROUP_OFFSETS; - private Map specificStartOffsets; - private Long startOffsetsTimestamp; -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaConsumerThread.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaConsumerThread.java deleted file mode 100644 index a24d3fe929b..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaConsumerThread.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.source; - -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorException; - -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; - -import java.util.Properties; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -public class KafkaConsumerThread implements Runnable { - - private final KafkaConsumer consumer; - private static final String CLIENT_ID_PREFIX = "seatunnel"; - private final ConsumerMetadata metadata; - - private final LinkedBlockingQueue>> tasks; - - public KafkaConsumerThread(ConsumerMetadata metadata) { - this.metadata = metadata; - this.tasks = new LinkedBlockingQueue<>(); - this.consumer = - initConsumer( - this.metadata.getBootstrapServers(), - this.metadata.getConsumerGroup(), - this.metadata.getProperties(), - !this.metadata.isCommitOnCheckpoint()); - } - - @Override - public void run() { - try { - while (!Thread.currentThread().isInterrupted()) { - try { - Consumer> task = tasks.poll(1, TimeUnit.SECONDS); - if (task != null) { - task.accept(consumer); - } - } catch (Exception e) { - throw new KafkaConnectorException( - KafkaConnectorErrorCode.CONSUME_THREAD_RUN_ERROR, e); - } - } - } finally { - try { - consumer.close(); - } catch (Throwable t) { - throw new KafkaConnectorException(KafkaConnectorErrorCode.CONSUMER_CLOSE_FAILED, t); - } - } - } - - public LinkedBlockingQueue>> getTasks() { - return tasks; - } - - private KafkaConsumer initConsumer( - String bootstrapServer, - String consumerGroup, - Properties properties, - boolean autoCommit) { - Properties props = new Properties(); - properties.forEach( - (key, value) -> props.setProperty(String.valueOf(key), String.valueOf(value))); - props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, consumerGroup); - props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer); - if (this.metadata.getProperties().get("client.id") == null) { - props.setProperty( - ConsumerConfig.CLIENT_ID_CONFIG, - CLIENT_ID_PREFIX + "-consumer-" + this.hashCode()); - } else { - props.setProperty( - ConsumerConfig.CLIENT_ID_CONFIG, - this.metadata.getProperties().get("client.id").toString()); - } - props.setProperty( - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - ByteArrayDeserializer.class.getName()); - props.setProperty( - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - ByteArrayDeserializer.class.getName()); - props.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, String.valueOf(autoCommit)); - - // Disable auto create topics feature - // props.setProperty(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "false"); - return new KafkaConsumer<>(props); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSource.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSource.java deleted file mode 100644 index 33e2792339a..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSource.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.source; - -import org.apache.seatunnel.api.common.JobContext; -import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.source.Boundedness; -import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.source.SourceReader; -import org.apache.seatunnel.api.source.SourceSplitEnumerator; -import org.apache.seatunnel.api.source.SupportParallelism; -import org.apache.seatunnel.api.table.catalog.CatalogTable; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.common.constants.JobMode; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaSourceState; - -import com.google.common.collect.Lists; - -import java.util.List; - -public class KafkaSource - implements SeaTunnelSource, - SupportParallelism { - - private JobContext jobContext; - - private final KafkaSourceConfig kafkaSourceConfig; - - public KafkaSource(ReadonlyConfig readonlyConfig) { - kafkaSourceConfig = new KafkaSourceConfig(readonlyConfig); - } - - @Override - public Boundedness getBoundedness() { - return JobMode.BATCH.equals(jobContext.getJobMode()) - ? Boundedness.BOUNDED - : Boundedness.UNBOUNDED; - } - - @Override - public String getPluginName() { - return org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.CONNECTOR_IDENTITY; - } - - @Override - public List getProducedCatalogTables() { - return Lists.newArrayList(kafkaSourceConfig.getCatalogTable()); - } - - @Override - public SourceReader createReader( - SourceReader.Context readerContext) { - return new KafkaSourceReader( - kafkaSourceConfig.getMetadata(), - kafkaSourceConfig.getDeserializationSchema(), - readerContext, - kafkaSourceConfig.getMessageFormatErrorHandleWay()); - } - - @Override - public SourceSplitEnumerator createEnumerator( - SourceSplitEnumerator.Context enumeratorContext) { - return new KafkaSourceSplitEnumerator( - kafkaSourceConfig.getMetadata(), - enumeratorContext, - kafkaSourceConfig.getDiscoveryIntervalMillis()); - } - - @Override - public SourceSplitEnumerator restoreEnumerator( - SourceSplitEnumerator.Context enumeratorContext, - KafkaSourceState checkpointState) { - return new KafkaSourceSplitEnumerator( - kafkaSourceConfig.getMetadata(), - enumeratorContext, - checkpointState, - kafkaSourceConfig.getDiscoveryIntervalMillis()); - } - - @Override - public void setJobContext(JobContext jobContext) { - this.jobContext = jobContext; - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceConfig.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceConfig.java deleted file mode 100644 index 5a1c335a798..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceConfig.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.source; - -import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.serialization.DeserializationSchema; -import org.apache.seatunnel.api.table.catalog.CatalogTable; -import org.apache.seatunnel.api.table.catalog.CatalogTableUtil; -import org.apache.seatunnel.api.table.catalog.PhysicalColumn; -import org.apache.seatunnel.api.table.catalog.TableIdentifier; -import org.apache.seatunnel.api.table.catalog.TableSchema; -import org.apache.seatunnel.api.table.catalog.schema.TableSchemaOptions; -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.MessageFormat; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.MessageFormatErrorHandleWay; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.StartMode; -import org.apache.seatunnel.format.compatible.kafka.connect.json.CompatibleKafkaConnectDeserializationSchema; -import org.apache.seatunnel.format.compatible.kafka.connect.json.KafkaConnectJsonFormatOptions; -import org.apache.seatunnel.format.json.JsonDeserializationSchema; -import org.apache.seatunnel.format.json.canal.CanalJsonDeserializationSchema; -import org.apache.seatunnel.format.json.debezium.DebeziumJsonDeserializationSchema; -import org.apache.seatunnel.format.json.exception.SeaTunnelJsonFormatException; -import org.apache.seatunnel.format.json.ogg.OggJsonDeserializationSchema; -import org.apache.seatunnel.format.text.TextDeserializationSchema; -import org.apache.seatunnel.format.text.constant.TextFormatConstant; - -import org.apache.commons.collections4.MapUtils; -import org.apache.kafka.common.TopicPartition; - -import lombok.Getter; - -import java.io.Serializable; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; - -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.BOOTSTRAP_SERVERS; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.COMMIT_ON_CHECKPOINT; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.CONNECTOR_IDENTITY; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.CONSUMER_GROUP; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.DEBEZIUM_RECORD_INCLUDE_SCHEMA; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.FIELD_DELIMITER; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.FORMAT; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.KAFKA_CONFIG; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.MESSAGE_FORMAT_ERROR_HANDLE_WAY_OPTION; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.PATTERN; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.START_MODE; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.START_MODE_OFFSETS; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.START_MODE_TIMESTAMP; -import static org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config.TOPIC; - -public class KafkaSourceConfig implements Serializable { - - private static final long serialVersionUID = 1L; - - @Getter private final ConsumerMetadata metadata; - - @Getter private final DeserializationSchema deserializationSchema; - - @Getter private final CatalogTable catalogTable; - - @Getter private final MessageFormatErrorHandleWay messageFormatErrorHandleWay; - - @Getter private final long discoveryIntervalMillis; - - public KafkaSourceConfig(ReadonlyConfig readonlyConfig) { - this.metadata = createConsumerMetadata(readonlyConfig); - this.discoveryIntervalMillis = readonlyConfig.get(KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS); - this.messageFormatErrorHandleWay = - readonlyConfig.get(MESSAGE_FORMAT_ERROR_HANDLE_WAY_OPTION); - this.catalogTable = createCatalogTable(readonlyConfig); - this.deserializationSchema = createDeserializationSchema(catalogTable, readonlyConfig); - } - - private ConsumerMetadata createConsumerMetadata(ReadonlyConfig readonlyConfig) { - ConsumerMetadata consumerMetadata = new ConsumerMetadata(); - consumerMetadata.setTopic(readonlyConfig.get(TOPIC)); - consumerMetadata.setBootstrapServers(readonlyConfig.get(BOOTSTRAP_SERVERS)); - consumerMetadata.setPattern(readonlyConfig.get(PATTERN)); - consumerMetadata.setProperties(new Properties()); - consumerMetadata.setConsumerGroup(readonlyConfig.get(CONSUMER_GROUP)); - consumerMetadata.setCommitOnCheckpoint(readonlyConfig.get(COMMIT_ON_CHECKPOINT)); - // parse start mode - readonlyConfig - .getOptional(START_MODE) - .ifPresent( - startMode -> { - consumerMetadata.setStartMode(startMode); - switch (startMode) { - case TIMESTAMP: - long startOffsetsTimestamp = - readonlyConfig.get(START_MODE_TIMESTAMP); - long currentTimestamp = System.currentTimeMillis(); - if (startOffsetsTimestamp < 0 - || startOffsetsTimestamp > currentTimestamp) { - throw new IllegalArgumentException( - "start_mode.timestamp The value is smaller than 0 or smaller than the current time"); - } - consumerMetadata.setStartOffsetsTimestamp( - startOffsetsTimestamp); - break; - case SPECIFIC_OFFSETS: - // Key is topic-partition, value is offset - Map offsetMap = - readonlyConfig.get(START_MODE_OFFSETS); - if (MapUtils.isEmpty(offsetMap)) { - throw new IllegalArgumentException( - "start mode is " - + StartMode.SPECIFIC_OFFSETS - + "but no specific offsets were specified."); - } - Map specificStartOffsets = - new HashMap<>(); - offsetMap.forEach( - (topicPartitionKey, offset) -> { - int splitIndex = topicPartitionKey.lastIndexOf("-"); - String topic = - topicPartitionKey.substring(0, splitIndex); - String partition = - topicPartitionKey.substring(splitIndex + 1); - TopicPartition topicPartition = - new TopicPartition( - topic, Integer.parseInt(partition)); - specificStartOffsets.put(topicPartition, offset); - }); - consumerMetadata.setSpecificStartOffsets(specificStartOffsets); - break; - default: - break; - } - }); - - readonlyConfig - .getOptional(KAFKA_CONFIG) - .ifPresent( - kafkaConfig -> - kafkaConfig.forEach( - (key, value) -> - consumerMetadata.getProperties().put(key, value))); - - return consumerMetadata; - } - - private CatalogTable createCatalogTable(ReadonlyConfig readonlyConfig) { - Optional> schemaOptions = - readonlyConfig.getOptional(TableSchemaOptions.SCHEMA); - if (schemaOptions.isPresent()) { - return CatalogTableUtil.buildWithConfig(readonlyConfig); - } else { - TableIdentifier tableIdentifier = TableIdentifier.of(CONNECTOR_IDENTITY, null, null); - TableSchema tableSchema = - TableSchema.builder() - .column( - PhysicalColumn.of( - "content", - new SeaTunnelRowType( - new String[] {"content"}, - new SeaTunnelDataType[] { - BasicType.STRING_TYPE - }), - 0, - false, - null, - null)) - .build(); - return CatalogTable.of( - tableIdentifier, - tableSchema, - Collections.emptyMap(), - Collections.emptyList(), - null); - } - } - - private DeserializationSchema createDeserializationSchema( - CatalogTable catalogTable, ReadonlyConfig readonlyConfig) { - SeaTunnelRowType seaTunnelRowType = catalogTable.getSeaTunnelRowType(); - - if (!readonlyConfig.getOptional(TableSchemaOptions.SCHEMA).isPresent()) { - return TextDeserializationSchema.builder() - .seaTunnelRowType(seaTunnelRowType) - .delimiter(TextFormatConstant.PLACEHOLDER) - .build(); - } - - MessageFormat format = readonlyConfig.get(FORMAT); - switch (format) { - case JSON: - return new JsonDeserializationSchema(false, false, seaTunnelRowType); - case TEXT: - String delimiter = readonlyConfig.get(FIELD_DELIMITER); - return TextDeserializationSchema.builder() - .seaTunnelRowType(seaTunnelRowType) - .delimiter(delimiter) - .build(); - case CANAL_JSON: - return CanalJsonDeserializationSchema.builder(seaTunnelRowType) - .setIgnoreParseErrors(true) - .build(); - case OGG_JSON: - return OggJsonDeserializationSchema.builder(seaTunnelRowType) - .setIgnoreParseErrors(true) - .build(); - case COMPATIBLE_KAFKA_CONNECT_JSON: - Boolean keySchemaEnable = - readonlyConfig.get( - KafkaConnectJsonFormatOptions.KEY_CONVERTER_SCHEMA_ENABLED); - Boolean valueSchemaEnable = - readonlyConfig.get( - KafkaConnectJsonFormatOptions.VALUE_CONVERTER_SCHEMA_ENABLED); - return new CompatibleKafkaConnectDeserializationSchema( - seaTunnelRowType, keySchemaEnable, valueSchemaEnable, false, false); - case DEBEZIUM_JSON: - boolean includeSchema = readonlyConfig.get(DEBEZIUM_RECORD_INCLUDE_SCHEMA); - return new DebeziumJsonDeserializationSchema(seaTunnelRowType, true, includeSchema); - default: - throw new SeaTunnelJsonFormatException( - CommonErrorCodeDeprecated.UNSUPPORTED_DATA_TYPE, - "Unsupported format: " + format); - } - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceFactory.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceFactory.java deleted file mode 100644 index 5ebf02f465c..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceFactory.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.source; - -import org.apache.seatunnel.api.configuration.util.OptionRule; -import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.source.SourceSplit; -import org.apache.seatunnel.api.table.connector.TableSource; -import org.apache.seatunnel.api.table.factory.Factory; -import org.apache.seatunnel.api.table.factory.TableSourceFactory; -import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.Config; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.StartMode; - -import com.google.auto.service.AutoService; - -import java.io.Serializable; - -@AutoService(Factory.class) -public class KafkaSourceFactory implements TableSourceFactory { - - @Override - public String factoryIdentifier() { - return "Kafka"; - } - - @Override - public OptionRule optionRule() { - return OptionRule.builder() - .required(Config.TOPIC, Config.BOOTSTRAP_SERVERS) - .optional( - Config.START_MODE, - Config.PATTERN, - Config.CONSUMER_GROUP, - Config.COMMIT_ON_CHECKPOINT, - Config.KAFKA_CONFIG, - Config.SCHEMA, - Config.FORMAT, - Config.DEBEZIUM_RECORD_INCLUDE_SCHEMA, - Config.KEY_PARTITION_DISCOVERY_INTERVAL_MILLIS) - .conditional(Config.START_MODE, StartMode.TIMESTAMP, Config.START_MODE_TIMESTAMP) - .conditional( - Config.START_MODE, StartMode.SPECIFIC_OFFSETS, Config.START_MODE_OFFSETS) - .build(); - } - - @Override - public - TableSource createSource(TableSourceFactoryContext context) { - return () -> (SeaTunnelSource) new KafkaSource(context.getOptions()); - } - - @Override - public Class getSourceClass() { - return KafkaSource.class; - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceReader.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceReader.java deleted file mode 100644 index 1e4a95d24b2..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceReader.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.source; - -import org.apache.seatunnel.api.serialization.DeserializationSchema; -import org.apache.seatunnel.api.source.Boundedness; -import org.apache.seatunnel.api.source.Collector; -import org.apache.seatunnel.api.source.SourceReader; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.config.MessageFormatErrorHandleWay; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorException; -import org.apache.seatunnel.format.compatible.kafka.connect.json.CompatibleKafkaConnectDeserializationSchema; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.serialization.StringDeserializer; - -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.time.Duration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.stream.Collectors; - -@Slf4j -public class KafkaSourceReader implements SourceReader { - - private static final long THREAD_WAIT_TIME = 500L; - private static final long POLL_TIMEOUT = 10000L; - - private final SourceReader.Context context; - private final ConsumerMetadata metadata; - private final Set sourceSplits; - private final Map> checkpointOffsetMap; - private final Map consumerThreadMap; - private final ExecutorService executorService; - private final DeserializationSchema deserializationSchema; - private final MessageFormatErrorHandleWay messageFormatErrorHandleWay; - - private final LinkedBlockingQueue pendingPartitionsQueue; - - private volatile boolean running = false; - - KafkaSourceReader( - ConsumerMetadata metadata, - DeserializationSchema deserializationSchema, - Context context, - MessageFormatErrorHandleWay messageFormatErrorHandleWay) { - this.metadata = metadata; - this.context = context; - this.messageFormatErrorHandleWay = messageFormatErrorHandleWay; - this.sourceSplits = new HashSet<>(); - this.deserializationSchema = deserializationSchema; - this.consumerThreadMap = new ConcurrentHashMap<>(); - this.checkpointOffsetMap = new ConcurrentHashMap<>(); - this.executorService = - Executors.newCachedThreadPool(r -> new Thread(r, "Kafka Source Data Consumer")); - pendingPartitionsQueue = new LinkedBlockingQueue<>(); - } - - @Override - public void open() {} - - @Override - public void close() throws IOException { - if (executorService != null) { - executorService.shutdownNow(); - } - } - - @Override - public void pollNext(Collector output) throws Exception { - if (!running) { - Thread.sleep(THREAD_WAIT_TIME); - return; - } - - while (pendingPartitionsQueue.size() != 0) { - sourceSplits.add(pendingPartitionsQueue.poll()); - } - sourceSplits.forEach( - sourceSplit -> - consumerThreadMap.computeIfAbsent( - sourceSplit.getTopicPartition(), - s -> { - KafkaConsumerThread thread = new KafkaConsumerThread(metadata); - executorService.submit(thread); - return thread; - })); - sourceSplits.forEach( - sourceSplit -> { - CompletableFuture completableFuture = new CompletableFuture<>(); - try { - consumerThreadMap - .get(sourceSplit.getTopicPartition()) - .getTasks() - .put( - consumer -> { - try { - Set partitions = - Sets.newHashSet( - sourceSplit.getTopicPartition()); - StringDeserializer stringDeserializer = - new StringDeserializer(); - stringDeserializer.configure( - Maps.fromProperties( - this.metadata.getProperties()), - false); - consumer.assign(partitions); - if (sourceSplit.getStartOffset() >= 0) { - consumer.seek( - sourceSplit.getTopicPartition(), - sourceSplit.getStartOffset()); - } - ConsumerRecords records = - consumer.poll( - Duration.ofMillis(POLL_TIMEOUT) - .toMillis()); - for (TopicPartition partition : partitions) { - List> - recordList = records.records(partition); - for (ConsumerRecord record : - recordList) { - - try { - if (deserializationSchema - instanceof - CompatibleKafkaConnectDeserializationSchema) { - ((CompatibleKafkaConnectDeserializationSchema) - deserializationSchema) - .deserialize( - record, output); - } else { - deserializationSchema.deserialize( - record.value(), output); - } - } catch (IOException e) { - if (this.messageFormatErrorHandleWay - == MessageFormatErrorHandleWay - .SKIP) { - log.warn( - "Deserialize message failed, skip this message, message: {}", - new String(record.value())); - continue; - } - throw e; - } - - if (Boundedness.BOUNDED.equals( - context.getBoundedness()) - && record.offset() - >= sourceSplit - .getEndOffset()) { - break; - } - } - long lastOffset = -1; - if (!recordList.isEmpty()) { - lastOffset = - recordList - .get(recordList.size() - 1) - .offset(); - sourceSplit.setStartOffset(lastOffset + 1); - } - - if (lastOffset >= sourceSplit.getEndOffset()) { - sourceSplit.setEndOffset(lastOffset); - } - } - } catch (Exception e) { - completableFuture.completeExceptionally(e); - } - completableFuture.complete(null); - }); - } catch (InterruptedException e) { - throw new KafkaConnectorException( - KafkaConnectorErrorCode.CONSUME_DATA_FAILED, e); - } - completableFuture.join(); - }); - - if (Boundedness.BOUNDED.equals(context.getBoundedness())) { - // signal to the source that we have reached the end of the data. - context.signalNoMoreElement(); - } - } - - @Override - public List snapshotState(long checkpointId) { - checkpointOffsetMap.put( - checkpointId, - sourceSplits.stream() - .collect( - Collectors.toMap( - KafkaSourceSplit::getTopicPartition, - KafkaSourceSplit::getStartOffset))); - return sourceSplits.stream().map(KafkaSourceSplit::copy).collect(Collectors.toList()); - } - - @Override - public void addSplits(List splits) { - running = true; - splits.forEach( - s -> { - try { - pendingPartitionsQueue.put(s); - } catch (InterruptedException e) { - throw new KafkaConnectorException( - KafkaConnectorErrorCode.ADD_SPLIT_CHECKPOINT_FAILED, e); - } - }); - } - - @Override - public void handleNoMoreSplits() { - log.info("receive no more splits message, this reader will not add new split."); - } - - @Override - public void notifyCheckpointComplete(long checkpointId) { - if (!checkpointOffsetMap.containsKey(checkpointId)) { - log.warn("checkpoint {} do not exist or have already been committed.", checkpointId); - } else { - checkpointOffsetMap - .remove(checkpointId) - .forEach( - (topicPartition, offset) -> { - try { - consumerThreadMap - .get(topicPartition) - .getTasks() - .put( - consumer -> { - if (this.metadata.isCommitOnCheckpoint()) { - Map - offsets = new HashMap<>(); - if (offset >= 0) { - offsets.put( - topicPartition, - new OffsetAndMetadata( - offset)); - consumer.commitSync(offsets); - } - } - }); - } catch (InterruptedException e) { - log.error("commit offset to kafka failed", e); - } - }); - } - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceSplit.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceSplit.java deleted file mode 100644 index 918557fa56b..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceSplit.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.source; - -import org.apache.seatunnel.api.source.SourceSplit; - -import org.apache.kafka.common.TopicPartition; - -import java.util.Objects; - -public class KafkaSourceSplit implements SourceSplit { - - private TopicPartition topicPartition; - private long startOffset = -1L; - private long endOffset = -1L; - - public KafkaSourceSplit(TopicPartition topicPartition) { - this.topicPartition = topicPartition; - } - - public KafkaSourceSplit(TopicPartition topicPartition, long startOffset, long endOffset) { - this.topicPartition = topicPartition; - this.startOffset = startOffset; - this.endOffset = endOffset; - } - - public long getStartOffset() { - return startOffset; - } - - public void setStartOffset(long startOffset) { - this.startOffset = startOffset; - } - - public long getEndOffset() { - return endOffset; - } - - public void setEndOffset(long endOffset) { - this.endOffset = endOffset; - } - - public TopicPartition getTopicPartition() { - return topicPartition; - } - - public void setTopicPartition(TopicPartition topicPartition) { - this.topicPartition = topicPartition; - } - - @Override - public String splitId() { - return topicPartition.topic() + "-" + topicPartition.partition(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - KafkaSourceSplit that = (KafkaSourceSplit) o; - return Objects.equals(topicPartition, that.topicPartition); - } - - @Override - public int hashCode() { - return Objects.hash(topicPartition); - } - - public KafkaSourceSplit copy() { - return new KafkaSourceSplit( - this.topicPartition, this.getStartOffset(), this.getEndOffset()); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceSplitEnumerator.java deleted file mode 100644 index e3ddebca587..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/source/KafkaSourceSplitEnumerator.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.source; - -import org.apache.seatunnel.api.source.SourceSplitEnumerator; -import org.apache.seatunnel.common.config.Common; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.exception.KafkaConnectorException; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.state.KafkaSourceState; - -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsOptions; -import org.apache.kafka.clients.admin.OffsetSpec; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.common.TopicPartition; - -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -@Slf4j -public class KafkaSourceSplitEnumerator - implements SourceSplitEnumerator { - - private static final String CLIENT_ID_PREFIX = "seatunnel"; - - private final ConsumerMetadata metadata; - private final Context context; - private long discoveryIntervalMillis; - private final AdminClient adminClient; - - private final Map pendingSplit; - private final Map assignedSplit; - private ScheduledExecutorService executor; - private ScheduledFuture scheduledFuture; - - KafkaSourceSplitEnumerator(ConsumerMetadata metadata, Context context) { - this.metadata = metadata; - this.context = context; - this.assignedSplit = new HashMap<>(); - this.pendingSplit = new HashMap<>(); - this.adminClient = initAdminClient(this.metadata.getProperties()); - } - - KafkaSourceSplitEnumerator( - ConsumerMetadata metadata, - Context context, - KafkaSourceState sourceState) { - this(metadata, context); - } - - KafkaSourceSplitEnumerator( - ConsumerMetadata metadata, - Context context, - long discoveryIntervalMillis) { - this(metadata, context); - this.discoveryIntervalMillis = discoveryIntervalMillis; - } - - KafkaSourceSplitEnumerator( - ConsumerMetadata metadata, - Context context, - KafkaSourceState sourceState, - long discoveryIntervalMillis) { - this(metadata, context, sourceState); - this.discoveryIntervalMillis = discoveryIntervalMillis; - } - - @Override - public void open() { - if (discoveryIntervalMillis > 0) { - this.executor = - Executors.newScheduledThreadPool( - 1, - runnable -> { - Thread thread = new Thread(runnable); - thread.setDaemon(true); - thread.setName("kafka-partition-dynamic-discovery"); - return thread; - }); - this.scheduledFuture = - executor.scheduleWithFixedDelay( - () -> { - try { - discoverySplits(); - } catch (Exception e) { - log.error("Dynamic discovery failure:", e); - } - }, - discoveryIntervalMillis, - discoveryIntervalMillis, - TimeUnit.MILLISECONDS); - } - } - - @Override - public void run() throws ExecutionException, InterruptedException { - fetchPendingPartitionSplit(); - setPartitionStartOffset(); - assignSplit(); - } - - private void setPartitionStartOffset() throws ExecutionException, InterruptedException { - Collection topicPartitions = pendingSplit.keySet(); - Map topicPartitionOffsets = null; - switch (metadata.getStartMode()) { - case EARLIEST: - topicPartitionOffsets = listOffsets(topicPartitions, OffsetSpec.earliest()); - break; - case GROUP_OFFSETS: - topicPartitionOffsets = listConsumerGroupOffsets(topicPartitions); - break; - case LATEST: - topicPartitionOffsets = listOffsets(topicPartitions, OffsetSpec.latest()); - break; - case TIMESTAMP: - topicPartitionOffsets = - listOffsets( - topicPartitions, - OffsetSpec.forTimestamp(metadata.getStartOffsetsTimestamp())); - break; - case SPECIFIC_OFFSETS: - topicPartitionOffsets = metadata.getSpecificStartOffsets(); - break; - default: - break; - } - topicPartitionOffsets.forEach( - (key, value) -> { - if (pendingSplit.containsKey(key)) { - pendingSplit.get(key).setStartOffset(value); - } - }); - } - - @Override - public void close() throws IOException { - if (this.adminClient != null) { - adminClient.close(); - } - if (scheduledFuture != null) { - scheduledFuture.cancel(false); - if (executor != null) { - executor.shutdownNow(); - } - } - } - - @Override - public void addSplitsBack(List splits, int subtaskId) { - if (!splits.isEmpty()) { - pendingSplit.putAll(convertToNextSplit(splits)); - } - } - - private Map convertToNextSplit( - List splits) { - try { - Map listOffsets = - listOffsets( - splits.stream() - .map(KafkaSourceSplit::getTopicPartition) - .filter(Objects::nonNull) - .collect(Collectors.toList()), - OffsetSpec.latest()); - splits.forEach( - split -> { - split.setStartOffset(split.getEndOffset() + 1); - split.setEndOffset(listOffsets.get(split.getTopicPartition())); - }); - return splits.stream() - .collect(Collectors.toMap(KafkaSourceSplit::getTopicPartition, split -> split)); - } catch (Exception e) { - throw new KafkaConnectorException( - KafkaConnectorErrorCode.ADD_SPLIT_BACK_TO_ENUMERATOR_FAILED, e); - } - } - - @Override - public int currentUnassignedSplitSize() { - return pendingSplit.size(); - } - - @Override - public void handleSplitRequest(int subtaskId) { - // Do nothing because Kafka source push split. - } - - @Override - public void registerReader(int subtaskId) { - if (!pendingSplit.isEmpty()) { - assignSplit(); - } - } - - @Override - public KafkaSourceState snapshotState(long checkpointId) throws Exception { - return new KafkaSourceState(new HashSet<>(assignedSplit.values())); - } - - @Override - public void notifyCheckpointComplete(long checkpointId) throws Exception { - // Do nothing - } - - private AdminClient initAdminClient(Properties properties) { - Properties props = new Properties(); - props.putAll(properties); - props.setProperty( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, this.metadata.getBootstrapServers()); - if (this.metadata.getProperties().get("client.id") == null) { - props.setProperty( - ConsumerConfig.CLIENT_ID_CONFIG, - CLIENT_ID_PREFIX + "-enumerator-admin-client-" + this.hashCode()); - } else { - props.setProperty( - ConsumerConfig.CLIENT_ID_CONFIG, - this.metadata.getProperties().get("client.id").toString()); - } - - return AdminClient.create(props); - } - - private Set getTopicInfo() throws ExecutionException, InterruptedException { - Collection topics; - if (this.metadata.isPattern()) { - Pattern pattern = Pattern.compile(this.metadata.getTopic()); - topics = - this.adminClient.listTopics().names().get().stream() - .filter(t -> pattern.matcher(t).matches()) - .collect(Collectors.toSet()); - } else { - topics = Arrays.asList(this.metadata.getTopic().split(",")); - } - log.info("Discovered topics: {}", topics); - - Collection partitions = - adminClient.describeTopics(topics).all().get().values().stream() - .flatMap( - t -> - t.partitions().stream() - .map( - p -> - new TopicPartition( - t.name(), p.partition()))) - .collect(Collectors.toSet()); - Map latestOffsets = listOffsets(partitions, OffsetSpec.latest()); - return partitions.stream() - .map( - partition -> { - KafkaSourceSplit split = new KafkaSourceSplit(partition); - split.setEndOffset(latestOffsets.get(split.getTopicPartition())); - return split; - }) - .collect(Collectors.toSet()); - } - - private synchronized void assignSplit() { - Map> readySplit = new HashMap<>(Common.COLLECTION_SIZE); - for (int taskID = 0; taskID < context.currentParallelism(); taskID++) { - readySplit.computeIfAbsent(taskID, id -> new ArrayList<>()); - } - - pendingSplit.forEach( - (key, value) -> { - if (!assignedSplit.containsKey(key)) { - readySplit.get(getSplitOwner(key, context.currentParallelism())).add(value); - } - }); - - readySplit.forEach( - (id, split) -> { - context.assignSplit(id, split); - if (discoveryIntervalMillis <= 0) { - context.signalNoMoreSplits(id); - } - }); - - assignedSplit.putAll(pendingSplit); - pendingSplit.clear(); - } - - private static int getSplitOwner(TopicPartition tp, int numReaders) { - int startIndex = ((tp.topic().hashCode() * 31) & 0x7FFFFFFF) % numReaders; - return (startIndex + tp.partition()) % numReaders; - } - - private Map listOffsets( - Collection partitions, OffsetSpec offsetSpec) - throws ExecutionException, InterruptedException { - Map topicPartitionOffsets = - partitions.stream() - .collect(Collectors.toMap(partition -> partition, __ -> offsetSpec)); - - return adminClient - .listOffsets(topicPartitionOffsets) - .all() - .thenApply( - result -> { - Map offsets = new HashMap<>(); - result.forEach( - (tp, offsetsResultInfo) -> { - if (offsetsResultInfo != null) { - offsets.put(tp, offsetsResultInfo.offset()); - } - }); - return offsets; - }) - .get(); - } - - public Map listConsumerGroupOffsets(Collection partitions) - throws ExecutionException, InterruptedException { - ListConsumerGroupOffsetsOptions options = - new ListConsumerGroupOffsetsOptions().topicPartitions(new ArrayList<>(partitions)); - return adminClient - .listConsumerGroupOffsets(metadata.getConsumerGroup(), options) - .partitionsToOffsetAndMetadata() - .thenApply( - result -> { - Map offsets = new HashMap<>(); - result.forEach( - (tp, oam) -> { - if (oam != null) { - offsets.put(tp, oam.offset()); - } - }); - return offsets; - }) - .get(); - } - - private void discoverySplits() throws ExecutionException, InterruptedException { - fetchPendingPartitionSplit(); - assignSplit(); - } - - private void fetchPendingPartitionSplit() throws ExecutionException, InterruptedException { - getTopicInfo() - .forEach( - split -> { - if (!assignedSplit.containsKey(split.getTopicPartition())) { - if (!pendingSplit.containsKey(split.getTopicPartition())) { - pendingSplit.put(split.getTopicPartition(), split); - } - } - }); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaAggregatedCommitInfo.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaAggregatedCommitInfo.java deleted file mode 100644 index 1f806e5f97b..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaAggregatedCommitInfo.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.state; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -@Data -@AllArgsConstructor -public class KafkaAggregatedCommitInfo implements Serializable { - List commitInfos; -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaCommitInfo.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaCommitInfo.java deleted file mode 100644 index 04600b55e0a..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaCommitInfo.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.state; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.io.Serializable; -import java.util.Properties; - -@Data -@AllArgsConstructor -public class KafkaCommitInfo implements Serializable { - - private final String transactionId; - private final Properties kafkaProperties; - private final long producerId; - private final short epoch; -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaSinkState.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaSinkState.java deleted file mode 100644 index dc26141ce5d..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaSinkState.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.state; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.io.Serializable; -import java.util.Properties; - -@Data -@AllArgsConstructor -public class KafkaSinkState implements Serializable { - - private final String transactionId; - private final String transactionIdPrefix; - private final long checkpointId; - private final Properties kafkaProperties; -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaSourceState.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaSourceState.java deleted file mode 100644 index 86140f95c9a..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/state/KafkaSourceState.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.state; - -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.source.KafkaSourceSplit; - -import java.io.Serializable; -import java.util.Set; - -public class KafkaSourceState implements Serializable { - - private Set assignedSplit; - - public KafkaSourceState(Set assignedSplit) { - this.assignedSplit = assignedSplit; - } - - public Set getAssignedSplit() { - return assignedSplit; - } - - public void setAssignedSplit(Set assignedSplit) { - this.assignedSplit = assignedSplit; - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/utils/KafkaSASLConstants.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/utils/KafkaSASLConstants.java deleted file mode 100644 index d5cf3dfdf36..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/utils/KafkaSASLConstants.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.utils; -/** - * @author patrick Created on 2022/6/7 15:51 - * @description kafka sasl认证常量 - */ -public class KafkaSASLConstants { - - /** 协议 */ - public static String protocol = "SASL_PLAINTEXT"; - /** 认证途径 */ - public static String saslMechanism = "PLAIN"; - /** 认证配置 */ - public static String saslJaasConfig = - "org.apache.kafka.common.security.plain.PlainLoginModule required\n" - + " username=\"kafka\"\n" - + " password=\"m33dt0lb6uAGR+4zgpR7jsR8YMIkXhQ=\";"; - - /** huawei配置:krb5.conf路径 */ - public static String krb5Config = ""; - - /** huawei配置：ZookeeperServerPrincipal */ - public static String zookeeperServerPrincipal = "zookeeper/hadoop.hadoop.com"; - - /** kafka连接地址 */ - public static String bootstrapServers = ""; - - /** kafka KERBEROS_DOMAIN_NAME 域名 */ - public static String kerberosDomainName = "hadoop.hadoop.com"; - - /** kafka KERBEROS_SERVER_NAME */ - public static String kerberosServerName = "kafka"; -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/utils/LoginUtil.java b/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/utils/LoginUtil.java deleted file mode 100644 index c3b82a378d4..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/hw/utils/LoginUtil.java +++ /dev/null @@ -1,169 +0,0 @@ -package org.apache.seatunnel.connectors.seatunnel.kafka.hw.utils; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; - -public class LoginUtil { - - public enum Module { - STORM("StormClient"), - KAFKA("KafkaClient"), - ZOOKEEPER("Client"); - - private String name; - - private Module(String name) { - this.name = name; - } - - public String getName() { - return name; - } - } - - /** line operator string */ - private static final String LINE_SEPARATOR = System.getProperty("line.separator"); - - /** jaas file postfix */ - private static final String JAAS_POSTFIX = ".jaas.conf"; - - /** is IBM jdk or not */ - private static final boolean IS_IBM_JDK = System.getProperty("java.vendor").contains("IBM"); - - /** IBM jdk login module */ - private static final String IBM_LOGIN_MODULE = - "com.ibm.security.auth.module.Krb5LoginModule required"; - - /** oracle jdk login module */ - private static final String SUN_LOGIN_MODULE = - "com.sun.security.auth.module.Krb5LoginModule required"; - - /** Zookeeper quorum principal. */ - public static final String ZOOKEEPER_AUTH_PRINCIPAL = "zookeeper.server.principal"; - - /** java security krb5 file path */ - public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; - - /** java security login file path */ - public static final String JAVA_SECURITY_LOGIN_CONF = "java.security.auth.login.config"; - - /** - * 设置jaas.conf文件 - * - * @param principal - * @param keytabPath - * @throws IOException - */ - public static void setJaasFile(String principal, String keytabPath) throws IOException { - String jaasPath = - new File(System.getProperty("java.io.tmpdir")) - + File.separator - + System.getProperty("user.name") - + JAAS_POSTFIX; - - // windows路径下分隔符替换 - jaasPath = jaasPath.replace("\\", "\\\\"); - // 删除jaas文件 - deleteJaasFile(jaasPath); - writeJaasFile(jaasPath, principal, keytabPath); - System.setProperty(JAVA_SECURITY_LOGIN_CONF, jaasPath); - } - - /** - * 设置zookeeper服务端principal - * - * @param zkServerPrincipal - * @throws IOException - */ - public static void setZookeeperServerPrincipal(String zkServerPrincipal) throws IOException { - System.setProperty(ZOOKEEPER_AUTH_PRINCIPAL, zkServerPrincipal); - String ret = System.getProperty(ZOOKEEPER_AUTH_PRINCIPAL); - if (ret == null) { - throw new IOException(ZOOKEEPER_AUTH_PRINCIPAL + " is null."); - } - if (!ret.equals(zkServerPrincipal)) { - throw new IOException( - ZOOKEEPER_AUTH_PRINCIPAL + " is " + ret + " is not " + zkServerPrincipal + "."); - } - } - - /** - * 设置krb5文件 - * - * @param krb5ConfFile - * @throws IOException - */ - public static void setKrb5Config(String krb5ConfFile) throws IOException { - System.setProperty(JAVA_SECURITY_KRB5_CONF, krb5ConfFile); - String ret = System.getProperty(JAVA_SECURITY_KRB5_CONF); - if (ret == null) { - throw new IOException(JAVA_SECURITY_KRB5_CONF + " is null."); - } - if (!ret.equals(krb5ConfFile)) { - throw new IOException( - JAVA_SECURITY_KRB5_CONF + " is " + ret + " is not " + krb5ConfFile + "."); - } - } - - /** - * 写入jaas文件 - * - * @throws IOException 写文件异常 - */ - private static void writeJaasFile(String jaasPath, String principal, String keytabPath) - throws IOException { - FileWriter writer = new FileWriter(new File(jaasPath)); - try { - writer.write(getJaasConfContext(principal, keytabPath)); - writer.flush(); - } catch (IOException e) { - throw new IOException("Failed to create jaas.conf File"); - } finally { - writer.close(); - } - } - - private static void deleteJaasFile(String jaasPath) throws IOException { - File jaasFile = new File(jaasPath); - if (jaasFile.exists()) { - if (!jaasFile.delete()) { - throw new IOException("Failed to delete exists jaas file."); - } - } - } - - private static String getJaasConfContext(String principal, String keytabPath) { - Module[] allModule = Module.values(); - StringBuilder builder = new StringBuilder(); - for (Module modlue : allModule) { - builder.append(getModuleContext(principal, keytabPath, modlue)); - } - return builder.toString(); - } - - private static String getModuleContext(String userPrincipal, String keyTabPath, Module module) { - StringBuilder builder = new StringBuilder(); - if (IS_IBM_JDK) { - builder.append(module.getName()).append(" {").append(LINE_SEPARATOR); - builder.append(IBM_LOGIN_MODULE).append(LINE_SEPARATOR); - builder.append("credsType=both").append(LINE_SEPARATOR); - builder.append("principal=\"" + userPrincipal + "\"").append(LINE_SEPARATOR); - builder.append("useKeytab=\"" + keyTabPath + "\"").append(LINE_SEPARATOR); - builder.append("debug=true;").append(LINE_SEPARATOR); - builder.append("};").append(LINE_SEPARATOR); - } else { - builder.append(module.getName()).append(" {").append(LINE_SEPARATOR); - builder.append(SUN_LOGIN_MODULE).append(LINE_SEPARATOR); - builder.append("useKeyTab=true").append(LINE_SEPARATOR); - builder.append("keyTab=\"" + keyTabPath + "\"").append(LINE_SEPARATOR); - builder.append("principal=\"" + userPrincipal + "\"").append(LINE_SEPARATOR); - builder.append("useTicketCache=false").append(LINE_SEPARATOR); - builder.append("storeKey=true").append(LINE_SEPARATOR); - builder.append("debug=true;").append(LINE_SEPARATOR); - builder.append("};").append(LINE_SEPARATOR); - } - - return builder.toString(); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/KafkaFactoryTest.java b/seatunnel-connectors-v2/connector-kafka-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/KafkaFactoryTest.java deleted file mode 100644 index 4ab96dc3078..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/KafkaFactoryTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka; - -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink.KafkaSinkFactory; -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.source.KafkaSourceFactory; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class KafkaFactoryTest { - - @Test - void optionRule() { - Assertions.assertNotNull((new KafkaSourceFactory()).optionRule()); - Assertions.assertNotNull((new KafkaSinkFactory()).optionRule()); - } -} diff --git a/seatunnel-connectors-v2/connector-kafka-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/KafkaStartOffsetTest.java b/seatunnel-connectors-v2/connector-kafka-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/KafkaStartOffsetTest.java deleted file mode 100644 index fd40fc49109..00000000000 --- a/seatunnel-connectors-v2/connector-kafka-hw/src/test/java/org/apache/seatunnel/connectors/seatunnel/kafka/KafkaStartOffsetTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.kafka; - -import org.apache.seatunnel.connectors.seatunnel.kafka.hw.sink.KafkaNoTransactionSender; - -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.config.SaslConfigs; -import org.apache.kafka.common.serialization.StringSerializer; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Properties; - -public class KafkaStartOffsetTest { - public static String saslJaasConfig = - "org.apache.kafka.common.security.plain.PlainLoginModule required\n" - + " username=\"kafka\"\n" - + " password=\"m33dt0lb6uAGR+4zgpR7jsR8YMIkXhQ=\";"; - - @Test - void getTopicNameAndPartition() { - String topicName = "my-topic-test"; - int partIndex = 1; - String key = "my-topic-test-1"; - int splitIndex = key.lastIndexOf("-"); - String topic = key.substring(0, splitIndex); - String partition = key.substring(splitIndex + 1); - Assertions.assertEquals(topic, topicName); - Assertions.assertEquals(Integer.valueOf(partition), partIndex); - } - - @Test - void producer() { - Properties props = new Properties(); - System.setProperty( - "java.security.krb5.conf", - "D:/env/hw-auth/kafkauser_1701846620406_keytab/krb5.conf"); - props.put( - SaslConfigs.SASL_JAAS_CONFIG, - "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab=\"D:/env/hw-auth/kafkauser_1701846620406_keytab/user.keytab\" principal=\"kafkauser\" debug=true;"); - props.put("bootstrap.servers", "10.68.120.90:21007,10.68.120.91:21007,10.68.120.92:21007"); - props.put("security.protocol", "SASL_PLAINTEXT"); - props.put("sasl.kerberos.service.name", "kafka"); - props.put("sasl.mechanism", "GSSAPI"); - props.put("key.serializer", StringSerializer.class.getName()); - props.put("value.serializer", StringSerializer.class.getName()); - - KafkaNoTransactionSender kafkaNoTransactionSender = new KafkaNoTransactionSender(props); - - for (int i = 0; i < 20; i++) { - kafkaNoTransactionSender.send( - new ProducerRecord("test-hw", "test huawei kafka!")); - } - } - - @Test - void consumer() { - Properties props = new Properties(); - System.setProperty( - "java.security.krb5.conf", - "D:/env/hw-auth/kafkauser_1701846620406_keytab/krb5.conf"); - props.put( - SaslConfigs.SASL_JAAS_CONFIG, - "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true storeKey=true keyTab=\"D:/env/hw-auth/kafkauser_1701846620406_keytab/user.keytab\" principal=\"kafkauser\" debug=true;"); - props.put("bootstrap.servers", "10.68.120.90:21007,10.68.120.91:21007,10.68.120.92:21007"); - props.put("security.protocol", "SASL_PLAINTEXT"); - props.put("sasl.kerberos.service.name", "kafka"); - props.put("sasl.mechanism", "GSSAPI"); - props.setProperty( - "key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); - props.setProperty( - "value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); - props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "test_hw"); - KafkaConsumer kafkaConsumer = new KafkaConsumer<>(props); - kafkaConsumer.subscribe(Arrays.asList("test-hw")); - for (int i = 0; i < 1000; i++) { - ConsumerRecords records = kafkaConsumer.poll(1000); - // 消息处理 - for (ConsumerRecord record : records) { - System.out.println( - "[NewConsumerExample], Received message: (" - + record.key() - + ", " - + record.value() - + ") at offset " - + record.offset()); - } - } - } -} diff --git a/seatunnel-connectors-v2/pom.xml b/seatunnel-connectors-v2/pom.xml index 7431da6523e..244c9ddc3fe 100644 --- a/seatunnel-connectors-v2/pom.xml +++ b/seatunnel-connectors-v2/pom.xml @@ -32,13 +32,13 @@ connector-common - + connector-clickhouse connector-jdbc - - connector-kafka-hw + connector-kafka + connector-hw-kafka @@ -49,7 +49,7 @@ - connector-elasticsearch-hw + connector-hw-elasticsearch diff --git a/seatunnel-examples/seatunnel-engine-examples/pom.xml b/seatunnel-examples/seatunnel-engine-examples/pom.xml index 3ede9cd67ca..50fbcc7057d 100644 --- a/seatunnel-examples/seatunnel-engine-examples/pom.xml +++ b/seatunnel-examples/seatunnel-engine-examples/pom.xml @@ -53,7 +53,17 @@ org.apache.seatunnel - connector-kafka-hw + connector-clickhouse + ${project.version} + + + org.apache.seatunnel + connector-hw-kafka + ${project.version} + + + org.apache.seatunnel + connector-kafka ${project.version} diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java index 39c8b3d9429..aa23f9e81a2 100644 --- a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java @@ -31,7 +31,7 @@ public class SeaTunnelEngineExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { - String configurePath = args.length > 0 ? args[0] : "/examples/hive_hive_hw.conf"; + String configurePath = args.length > 0 ? args[0] : "/examples/clickhouse_kafka.conf"; String configFile = getTestConfigFile(configurePath); ClientCommandArgs clientCommandArgs = new ClientCommandArgs(); clientCommandArgs.setConfigFile(configFile);