From 770bade75ba9dc6f04bea75e5cb9c567a7401a14 Mon Sep 17 00:00:00 2001 From: Kirs Date: Fri, 1 Jul 2022 16:24:54 +0800 Subject: [PATCH 01/76] Update .asf.yaml (#2103) --- .asf.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 8ca64fdcb3d..15feb91b4db 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -29,7 +29,7 @@ github: - etl-framework enabled_merge_buttons: squash: true - merge: false + merge: true rebase: false protected_branches: dev: @@ -44,4 +44,4 @@ notifications: issues: commits@seatunnel.apache.org pullrequests: commits@seatunnel.apache.org pullrequests_status: commits@seatunnel.apache.org - pullrequests_comment: commits@seatunnel.apache.org \ No newline at end of file + pullrequests_comment: commits@seatunnel.apache.org From 19867ee6ad8139f0529ac7a49b782aab7f58ee56 Mon Sep 17 00:00:00 2001 From: Kirs Date: Fri, 1 Jul 2022 16:33:15 +0800 Subject: [PATCH 02/76] Revert "Update .asf.yaml (#2103)" (#2104) This reverts commit 770bade75ba9dc6f04bea75e5cb9c567a7401a14. --- .asf.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 15feb91b4db..8ca64fdcb3d 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -29,7 +29,7 @@ github: - etl-framework enabled_merge_buttons: squash: true - merge: true + merge: false rebase: false protected_branches: dev: @@ -44,4 +44,4 @@ notifications: issues: commits@seatunnel.apache.org pullrequests: commits@seatunnel.apache.org pullrequests_status: commits@seatunnel.apache.org - pullrequests_comment: commits@seatunnel.apache.org + pullrequests_comment: commits@seatunnel.apache.org \ No newline at end of file From a505d50f6c7151a612e974bba28927d5faac021e Mon Sep 17 00:00:00 2001 From: Kirs Date: Fri, 1 Jul 2022 18:19:28 +0800 Subject: [PATCH 03/76] Ignore some dead-link check (#2108) * Ignore some dead-link check * Update .dlc.json --- .dlc.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dlc.json b/.dlc.json index db4f4d1f8b1..8c7ba9ed05e 100644 --- a/.dlc.json +++ b/.dlc.json @@ -17,6 +17,9 @@ }, { "pattern": "^/docs/category" + }, + { + "pattern": "^https://opencollective.com" } ], "timeout": "10s", From cd34f1d388647ae917d70e7feb9c8b999a35091f Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 1 Jul 2022 21:57:51 +0800 Subject: [PATCH 04/76] [Connector-V2]unify connector v2 modules name (#2106) * unify connector v2 modules name * add codeql check timeout from 30min to 60min --- .github/workflows/codeql.yaml | 2 +- .../sink/Assert.md | 0 .../sink/Clickhouse.md | 0 .../sink/ClickhouseFile.md | 0 .../sink/Jdbc.md | 0 .../sink/common-options.md | 0 .../source/Clickhouse.md | 0 .../source/Jdbc.md | 0 .../source/common-options.md | 0 seatunnel-e2e/pom.xml | 4 ++-- .../pom.xml | 2 +- .../seatunnel/e2e/flink/FlinkContainer.java | 0 .../flink/assertion/FakeSourceToAssertIT.java | 0 .../e2e/flink/fake/FakeSourceToConsoleIT.java | 0 .../assertion/fakesource_to_assert.conf | 0 .../resources/fake/fakesource_to_console.conf | 0 .../src/test/resources/log4j.properties | 23 ------------------- .../pom.xml | 2 +- .../seatunnel/e2e/spark/SparkContainer.java | 0 .../e2e/spark/fake/FakeSourceToConsoleIT.java | 0 .../resources/fake/fakesource_to_console.conf | 0 .../src/test/resources/log4j.properties | 22 ------------------ 22 files changed, 5 insertions(+), 50 deletions(-) rename docs/en/{new-connector => connector-v2}/sink/Assert.md (100%) rename docs/en/{new-connector => connector-v2}/sink/Clickhouse.md (100%) rename docs/en/{new-connector => connector-v2}/sink/ClickhouseFile.md (100%) rename docs/en/{new-connector => connector-v2}/sink/Jdbc.md (100%) rename docs/en/{new-connector => connector-v2}/sink/common-options.md (100%) rename docs/en/{new-connector => connector-v2}/source/Clickhouse.md (100%) rename docs/en/{new-connector => connector-v2}/source/Jdbc.md (100%) rename docs/en/{new-connector => connector-v2}/source/common-options.md (100%) rename seatunnel-e2e/{seatunnel-flink-new-connector-e2e => seatunnel-flink-connector-v2-e2e}/pom.xml (96%) rename seatunnel-e2e/{seatunnel-flink-new-connector-e2e => seatunnel-flink-connector-v2-e2e}/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java (100%) rename seatunnel-e2e/{seatunnel-flink-new-connector-e2e => seatunnel-flink-connector-v2-e2e}/src/test/java/org/apache/seatunnel/e2e/flink/assertion/FakeSourceToAssertIT.java (100%) rename seatunnel-e2e/{seatunnel-flink-new-connector-e2e => seatunnel-flink-connector-v2-e2e}/src/test/java/org/apache/seatunnel/e2e/flink/fake/FakeSourceToConsoleIT.java (100%) rename seatunnel-e2e/{seatunnel-flink-new-connector-e2e => seatunnel-flink-connector-v2-e2e}/src/test/resources/assertion/fakesource_to_assert.conf (100%) rename seatunnel-e2e/{seatunnel-flink-new-connector-e2e => seatunnel-flink-connector-v2-e2e}/src/test/resources/fake/fakesource_to_console.conf (100%) delete mode 100644 seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/resources/log4j.properties rename seatunnel-e2e/{seatunnel-spark-new-connector-e2e => seatunnel-spark-connector-v2-e2e}/pom.xml (96%) rename seatunnel-e2e/{seatunnel-spark-new-connector-e2e => seatunnel-spark-connector-v2-e2e}/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java (100%) rename seatunnel-e2e/{seatunnel-spark-new-connector-e2e => seatunnel-spark-connector-v2-e2e}/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java (100%) rename seatunnel-e2e/{seatunnel-spark-new-connector-e2e => seatunnel-spark-connector-v2-e2e}/src/test/resources/fake/fakesource_to_console.conf (100%) delete mode 100644 seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/resources/log4j.properties diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 28b656fad53..9fa10da508b 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -27,7 +27,7 @@ jobs: analyze: name: Analyze runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 env: JAVA_TOOL_OPTIONS: -Xmx2G -Xms2G -Dhttp.keepAlive=false -Dmaven.test.skip=true -Dcheckstyle.skip=true -Dlicense.skipAddThirdParty=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 diff --git a/docs/en/new-connector/sink/Assert.md b/docs/en/connector-v2/sink/Assert.md similarity index 100% rename from docs/en/new-connector/sink/Assert.md rename to docs/en/connector-v2/sink/Assert.md diff --git a/docs/en/new-connector/sink/Clickhouse.md b/docs/en/connector-v2/sink/Clickhouse.md similarity index 100% rename from docs/en/new-connector/sink/Clickhouse.md rename to docs/en/connector-v2/sink/Clickhouse.md diff --git a/docs/en/new-connector/sink/ClickhouseFile.md b/docs/en/connector-v2/sink/ClickhouseFile.md similarity index 100% rename from docs/en/new-connector/sink/ClickhouseFile.md rename to docs/en/connector-v2/sink/ClickhouseFile.md diff --git a/docs/en/new-connector/sink/Jdbc.md b/docs/en/connector-v2/sink/Jdbc.md similarity index 100% rename from docs/en/new-connector/sink/Jdbc.md rename to docs/en/connector-v2/sink/Jdbc.md diff --git a/docs/en/new-connector/sink/common-options.md b/docs/en/connector-v2/sink/common-options.md similarity index 100% rename from docs/en/new-connector/sink/common-options.md rename to docs/en/connector-v2/sink/common-options.md diff --git a/docs/en/new-connector/source/Clickhouse.md b/docs/en/connector-v2/source/Clickhouse.md similarity index 100% rename from docs/en/new-connector/source/Clickhouse.md rename to docs/en/connector-v2/source/Clickhouse.md diff --git a/docs/en/new-connector/source/Jdbc.md b/docs/en/connector-v2/source/Jdbc.md similarity index 100% rename from docs/en/new-connector/source/Jdbc.md rename to docs/en/connector-v2/source/Jdbc.md diff --git a/docs/en/new-connector/source/common-options.md b/docs/en/connector-v2/source/common-options.md similarity index 100% rename from docs/en/new-connector/source/common-options.md rename to docs/en/connector-v2/source/common-options.md diff --git a/seatunnel-e2e/pom.xml b/seatunnel-e2e/pom.xml index 4c0b0fd0b81..c74e5c31abf 100644 --- a/seatunnel-e2e/pom.xml +++ b/seatunnel-e2e/pom.xml @@ -29,8 +29,8 @@ seatunnel-flink-e2e seatunnel-spark-e2e - seatunnel-flink-new-connector-e2e - seatunnel-spark-new-connector-e2e + seatunnel-flink-connector-v2-e2e + seatunnel-spark-connector-v2-e2e seatunnel-flink-sql-e2e diff --git a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/pom.xml b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml similarity index 96% rename from seatunnel-e2e/seatunnel-flink-new-connector-e2e/pom.xml rename to seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml index 5c59d849102..c494f80f886 100644 --- a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/pom.xml @@ -23,7 +23,7 @@ 4.0.0 - seatunnel-flink-new-connector-e2e + seatunnel-flink-connector-v2-e2e diff --git a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java similarity index 100% rename from seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java rename to seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java diff --git a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/flink/assertion/FakeSourceToAssertIT.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/assertion/FakeSourceToAssertIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/flink/assertion/FakeSourceToAssertIT.java rename to seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/assertion/FakeSourceToAssertIT.java diff --git a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/flink/fake/FakeSourceToConsoleIT.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/fake/FakeSourceToConsoleIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/flink/fake/FakeSourceToConsoleIT.java rename to seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/fake/FakeSourceToConsoleIT.java diff --git a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf similarity index 100% rename from seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/resources/assertion/fakesource_to_assert.conf rename to seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf diff --git a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf similarity index 100% rename from seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/resources/fake/fakesource_to_console.conf rename to seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf diff --git a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/resources/log4j.properties b/seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/resources/log4j.properties deleted file mode 100644 index 57b61a3c141..00000000000 --- a/seatunnel-e2e/seatunnel-flink-new-connector-e2e/src/test/resources/log4j.properties +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Set everything to be logged to the console -log4j.rootCategory=ERROR, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n diff --git a/seatunnel-e2e/seatunnel-spark-new-connector-e2e/pom.xml b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml similarity index 96% rename from seatunnel-e2e/seatunnel-spark-new-connector-e2e/pom.xml rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml index 925075cbb18..732d9e51bf6 100644 --- a/seatunnel-e2e/seatunnel-spark-new-connector-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/pom.xml @@ -24,7 +24,7 @@ 4.0.0 jar - seatunnel-spark-new-connector-e2e + seatunnel-spark-connector-v2-e2e diff --git a/seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java similarity index 100% rename from seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java diff --git a/seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/fake/FakeSourceToConsoleIT.java diff --git a/seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf similarity index 100% rename from seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/resources/fake/fakesource_to_console.conf rename to seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf diff --git a/seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/resources/log4j.properties b/seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/resources/log4j.properties deleted file mode 100644 index 89547981c79..00000000000 --- a/seatunnel-e2e/seatunnel-spark-new-connector-e2e/src/test/resources/log4j.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Set everything to be logged to the console -log4j.rootCategory=ERROR, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n From 7ec0ada2b93d8fd5be103bb5b44acfc733798e0a Mon Sep 17 00:00:00 2001 From: ic4y <83933160+ic4y@users.noreply.github.com> Date: Fri, 1 Jul 2022 21:58:37 +0800 Subject: [PATCH 05/76] [bugfix] Check isOpen before closing (#2107) --- .../seatunnel/jdbc/sink/JdbcSinkAggregatedCommitter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkAggregatedCommitter.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkAggregatedCommitter.java index f0ee2671eda..d142e95bbe2 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkAggregatedCommitter.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/sink/JdbcSinkAggregatedCommitter.java @@ -83,7 +83,9 @@ public void abort(List aggregatedCommitInfo) throws IO public void close() throws IOException { try { - xaFacade.close(); + if (xaFacade.isOpen()) { + xaFacade.close(); + } } catch (Exception e) { ExceptionUtils.rethrowIOException(e); } From 66f8161ababc5730cf8822292ca384f9612ff5a8 Mon Sep 17 00:00:00 2001 From: songjianet <1778651752@qq.com> Date: Sun, 3 Jul 2022 10:30:43 +0800 Subject: [PATCH 06/76] [Feat][UI] Add multi-language related configuration. (#2116) --- seatunnel-ui/src/App.tsx | 13 +++++++-- seatunnel-ui/src/locales/en_US/index.ts | 24 ++++++++++++++++ seatunnel-ui/src/locales/index.ts | 32 +++++++++++++++++++++ seatunnel-ui/src/locales/zh_CN/index.ts | 24 ++++++++++++++++ seatunnel-ui/src/main.ts | 8 ++++++ seatunnel-ui/src/store/locales/index.ts | 37 +++++++++++++++++++++++++ seatunnel-ui/src/store/locales/types.ts | 24 ++++++++++++++++ 7 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 seatunnel-ui/src/locales/en_US/index.ts create mode 100644 seatunnel-ui/src/locales/index.ts create mode 100644 seatunnel-ui/src/locales/zh_CN/index.ts create mode 100644 seatunnel-ui/src/store/locales/index.ts create mode 100644 seatunnel-ui/src/store/locales/types.ts diff --git a/seatunnel-ui/src/App.tsx b/seatunnel-ui/src/App.tsx index 2a5ea199669..7ceca19d9a1 100644 --- a/seatunnel-ui/src/App.tsx +++ b/seatunnel-ui/src/App.tsx @@ -17,12 +17,19 @@ import { defineComponent } from 'vue' import { NButton } from 'naive-ui' +import { useI18n } from 'vue-i18n' - const App = defineComponent({ - setup() {}, +const App = defineComponent({ + setup() { + const { t } = useI18n() + + return { t } + }, render() { + const { t } = this + return ( - test + {t('test.test')} ) } }) diff --git a/seatunnel-ui/src/locales/en_US/index.ts b/seatunnel-ui/src/locales/en_US/index.ts new file mode 100644 index 00000000000..1e7d2e414b3 --- /dev/null +++ b/seatunnel-ui/src/locales/en_US/index.ts @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const test = { + 'test': 'test' +} + +export default { + test +} \ No newline at end of file diff --git a/seatunnel-ui/src/locales/index.ts b/seatunnel-ui/src/locales/index.ts new file mode 100644 index 00000000000..1d2769c173e --- /dev/null +++ b/seatunnel-ui/src/locales/index.ts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { createI18n } from 'vue-i18n' +import zh_CN from './zh_CN' +import en_US from './en_US' + +const i18n = createI18n({ + legacy: false, + globalInjection: true, + locale: 'en_US', + messages: { + zh_CN, + en_US + } +}) + +export default i18n \ No newline at end of file diff --git a/seatunnel-ui/src/locales/zh_CN/index.ts b/seatunnel-ui/src/locales/zh_CN/index.ts new file mode 100644 index 00000000000..e979946bb55 --- /dev/null +++ b/seatunnel-ui/src/locales/zh_CN/index.ts @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const test = { + 'test': '测试' +} + +export default { + test +} \ No newline at end of file diff --git a/seatunnel-ui/src/main.ts b/seatunnel-ui/src/main.ts index 8d92c860499..f5163bb6a0a 100644 --- a/seatunnel-ui/src/main.ts +++ b/seatunnel-ui/src/main.ts @@ -17,7 +17,15 @@ import { createApp } from 'vue' import App from './App' +import { createPinia } from 'pinia' +import piniaPluginPersistedstate from 'pinia-plugin-persistedstate' +import i18n from '@/locales' const app = createApp(App) +const pinia = createPinia() +pinia.use(piniaPluginPersistedstate) + +app.use(pinia) +app.use(i18n) app.mount('#app') \ No newline at end of file diff --git a/seatunnel-ui/src/store/locales/index.ts b/seatunnel-ui/src/store/locales/index.ts new file mode 100644 index 00000000000..b5139aa1c9c --- /dev/null +++ b/seatunnel-ui/src/store/locales/index.ts @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { defineStore } from 'pinia' +import { LocalesStore, Locales } from './types' + +export const useLocalesStore = defineStore({ + id: 'locales', + state: (): LocalesStore => ({ + locales: 'en_US' + }), + persist: true, + getters: { + getLocales(): Locales { + return this.locales + } + }, + actions: { + setLocales(lang: Locales): void { + this.locales = lang + } + } +}) \ No newline at end of file diff --git a/seatunnel-ui/src/store/locales/types.ts b/seatunnel-ui/src/store/locales/types.ts new file mode 100644 index 00000000000..095d54269ad --- /dev/null +++ b/seatunnel-ui/src/store/locales/types.ts @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +type Locales = 'zh_CN' | 'en_US' + +interface LocalesStore { + locales: Locales +} + +export { LocalesStore, Locales } \ No newline at end of file From 528c6fdeae34763193ca923099c88edcad5904d7 Mon Sep 17 00:00:00 2001 From: gleiyu Date: Sun, 3 Jul 2022 12:51:44 +0800 Subject: [PATCH 07/76] [Bug] [seatunnel-connector-flink-fake] Type is not supported: BigInteger (#2118) * remove biginteger --- docs/en/connector/source/Fake.mdx | 2 +- .../flink/fake/source/MockSchema.java | 19 ++++++------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/docs/en/connector/source/Fake.mdx b/docs/en/connector/source/Fake.mdx index 4487f828181..12a6c2bfb7f 100644 --- a/docs/en/connector/source/Fake.mdx +++ b/docs/en/connector/source/Fake.mdx @@ -100,7 +100,7 @@ column_config option type. | name | type | required | default value | support values | |-------------|-------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | name | string | yes | string | - | -| type | string | yes | string | int,integer,byte,boolean,char,
character,short,long,float,double,
date,timestamp,decimal,bigdecimal,
bigint,biginteger,int[],byte[],
boolean[],char[],character[],short[],
long[],float[],double[],string[],
binary,varchar | +| type | string | yes | string | int,integer,byte,boolean,char,
character,short,long,float,double,
date,timestamp,decimal,bigdecimal,
bigint,int[],byte[],
boolean[],char[],character[],short[],
long[],float[],double[],string[],
binary,varchar | | mock_config | mock_config | no | - | - | mock_config Option diff --git a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/source/MockSchema.java b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/source/MockSchema.java index b8f0428d1a8..37f1b33726f 100644 --- a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/source/MockSchema.java +++ b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-fake/src/main/java/org/apache/seatunnel/flink/fake/source/MockSchema.java @@ -53,7 +53,6 @@ import java.io.ByteArrayInputStream; import java.io.Serializable; import java.math.BigDecimal; -import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.util.ArrayList; @@ -132,6 +131,7 @@ public TypeInformation typeInformation() { dataType = BasicTypeInfo.SHORT_TYPE_INFO; break; case "long": + case "bigint": dataType = BasicTypeInfo.LONG_TYPE_INFO; break; case "float": @@ -150,10 +150,6 @@ public TypeInformation typeInformation() { case "bigdecimal": dataType = BasicTypeInfo.BIG_DEC_TYPE_INFO; break; - case "bigint": - case "biginteger": - dataType = BasicTypeInfo.BIG_INT_TYPE_INFO; - break; case "int[]": dataType = PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO; break; @@ -193,11 +189,11 @@ public TypeInformation typeInformation() { return dataType; } - public Object mockData(){ + public Object mockData() { Object mockData; MockConfig mockConfig = new MockConfig(); resolve(mockConfig); - switch (this.type.trim().toLowerCase()){ + switch (this.type.trim().toLowerCase()) { case "int": case "integer": mockData = JMockData.mock(int.class, mockConfig); @@ -216,6 +212,7 @@ public Object mockData(){ mockData = JMockData.mock(short.class, mockConfig); break; case "long": + case "bigint": mockData = JMockData.mock(long.class, mockConfig); break; case "float": @@ -234,10 +231,6 @@ public Object mockData(){ case "bigdecimal": mockData = JMockData.mock(BigDecimal.class, mockConfig); break; - case "bigint": - case "biginteger": - mockData = JMockData.mock(BigInteger.class, mockConfig); - break; case "int[]": mockData = JMockData.mock(int[].class, mockConfig); break; @@ -426,7 +419,7 @@ public static RowTypeInfo mockRowTypeInfo(List mockDataSchema) { return new RowTypeInfo(types, fieldNames); } - public static Row mockRowData(List mockDataSchema){ + public static Row mockRowData(List mockDataSchema) { Object[] fieldByPosition = new Object[mockDataSchema.size()]; for (int index = 0; index < mockDataSchema.size(); index++) { MockSchema schema = mockDataSchema.get(index); @@ -470,7 +463,7 @@ public static Row mockRowData(List mockDataSchema){ DEFAULT_MOCK_SCHEMAS.add(ageSchema); } - public static List resolveConfig(Config config){ + public static List resolveConfig(Config config) { if (config.hasPath(MOCK_DATA_SCHEMA)) { return config.getConfigList(MOCK_DATA_SCHEMA) .stream() From 418e40818d108c4b4330061dde688ac9eb9f8363 Mon Sep 17 00:00:00 2001 From: Hong Liu <844981280@qq.com> Date: Mon, 4 Jul 2022 10:47:34 +0800 Subject: [PATCH 08/76] Fix the bug Unrecognized field "TwoPhaseCommit" after doris 0.15 (#2054) Co-authored-by: smallhibiscus <844981280> --- .../seatunnel/flink/doris/sink/RespContent.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-doris/src/main/java/org/apache/seatunnel/flink/doris/sink/RespContent.java b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-doris/src/main/java/org/apache/seatunnel/flink/doris/sink/RespContent.java index 2f18861172b..67bf3781e2a 100644 --- a/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-doris/src/main/java/org/apache/seatunnel/flink/doris/sink/RespContent.java +++ b/seatunnel-connectors/seatunnel-connectors-flink/seatunnel-connector-flink-doris/src/main/java/org/apache/seatunnel/flink/doris/sink/RespContent.java @@ -38,6 +38,9 @@ public class RespContent implements Serializable { */ @JsonProperty("Label") private String label; + + @JsonProperty("TwoPhaseCommit") + private String twoPhaseCommit; /** * Import complete status. * "Success": Indicates that the import was successful. @@ -140,6 +143,14 @@ public void setLabel(String label) { this.label = label; } + public String getTwoPhaseCommit() { + return twoPhaseCommit; + } + + public void setTwoPhaseCommit(String twoPhaseCommit) { + this.twoPhaseCommit = twoPhaseCommit; + } + public String getStatus() { return status; } @@ -265,6 +276,7 @@ public String toString() { return "RespContent{" + "txnId=" + txnId + ", label='" + label + '\'' + + ", twoPhaseCommit='" + twoPhaseCommit + '\'' + ", status='" + status + '\'' + ", existingJobStatus='" + existingJobStatus + '\'' + ", message='" + message + '\'' + From da4ad1745fb742cb5c0f5268884782d027bafce2 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 4 Jul 2022 11:49:58 +0800 Subject: [PATCH 09/76] ignore backend ci for ui update (#2121) --- .github/workflows/backend.yml | 1 + .github/workflows/code-analysys.yml | 1 + .github/workflows/codeql.yaml | 1 + .github/workflows/docker.yml | 1 + .github/workflows/publish-docker.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 8d4242a8b1d..db406b3937e 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -23,6 +23,7 @@ on: paths-ignore: - 'docs/**' - '**/*.md' + - 'seatunnel-ui/**' concurrency: group: backend-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/code-analysys.yml b/.github/workflows/code-analysys.yml index 0211b5603ff..3032c3428e7 100644 --- a/.github/workflows/code-analysys.yml +++ b/.github/workflows/code-analysys.yml @@ -21,6 +21,7 @@ on: paths-ignore: - 'docs/**' - '**/*.md' + - 'seatunnel-ui/**' jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 9fa10da508b..237bebd08e1 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -22,6 +22,7 @@ on: paths-ignore: - 'docs/**' - '**/*.md' + - 'seatunnel-ui/**' jobs: analyze: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7d31f392f36..a263acd68f3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,6 +25,7 @@ on: paths-ignore: - 'docs/**' - '**/*.md' + - 'seatunnel-ui/**' concurrency: group: docker-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index a7a53d24cde..c2ffb5b433a 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -24,6 +24,7 @@ on: paths-ignore: - 'docs/**' - '**/*.md' + - 'seatunnel-ui/**' env: HUB: ghcr.io/${{ github.repository }} From ffcf3f59e2bcbc366e018c8a9b4a324067ab62fa Mon Sep 17 00:00:00 2001 From: Kirs Date: Mon, 4 Jul 2022 17:22:14 +0800 Subject: [PATCH 10/76] [Connector-V2]Hive Source (#2123) * [Connector-V2]Hive Source * Unified dep version * Unified dep version * Add License and Notice --- plugin-mapping.properties | 1 + pom.xml | 181 +++++++++++- .../api/common/PrepareFailException.java | 5 + .../connector-hive/pom.xml | 25 ++ .../seatunnel/hive/config/SourceConfig.java | 29 ++ .../hive/exception/HivePluginException.java | 29 ++ .../seatunnel/hive/sink/HiveSinkConfig.java | 2 +- .../seatunnel/hive/source/HadoopConf.java | 34 +++ .../seatunnel/hive/source/HiveSource.java | 129 +++++++++ .../hive/source/HiveSourceReader.java | 98 +++++++ .../hive/source/HiveSourceSplit.java | 36 +++ .../source/HiveSourceSplitEnumerator.java | 126 +++++++++ .../hive/source/HiveSourceState.java | 35 +++ .../reader/format/AbstractReadStrategy.java | 82 ++++++ .../file/reader/format/OrcReadStrategy.java | 123 +++++++++ .../file/reader/format/ReadStrategy.java | 42 +++ .../file/reader/format/TextReadStrategy.java | 56 ++++ seatunnel-dependency-shade/pom.xml | 39 +++ .../seatunnel-hive-shade/pom.xml | 69 +++++ seatunnel-dist/release-docs/LICENSE | 89 ++++-- seatunnel-dist/release-docs/NOTICE | 140 ++++++++++ .../release-docs/licenses/LICENSE-ant.txt | 48 ++++ .../release-docs/licenses/LICENSE-ivy.txt | 259 ++++++++++++++++++ .../LICENSE-parquet-hadoop-bundle.txt | 248 +++++++++++++++++ .../release-docs/licenses/LICENSE-st.txt | 26 ++ tools/dependencies/known-dependencies.txt | 74 +++-- 26 files changed, 1982 insertions(+), 43 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/config/SourceConfig.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/exception/HivePluginException.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HadoopConf.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSource.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceReader.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceSplit.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceSplitEnumerator.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceState.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/AbstractReadStrategy.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/OrcReadStrategy.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/ReadStrategy.java create mode 100644 seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/TextReadStrategy.java create mode 100644 seatunnel-dependency-shade/pom.xml create mode 100644 seatunnel-dependency-shade/seatunnel-hive-shade/pom.xml create mode 100644 seatunnel-dist/release-docs/licenses/LICENSE-ant.txt create mode 100644 seatunnel-dist/release-docs/licenses/LICENSE-ivy.txt create mode 100644 seatunnel-dist/release-docs/licenses/LICENSE-parquet-hadoop-bundle.txt create mode 100644 seatunnel-dist/release-docs/licenses/LICENSE-st.txt diff --git a/plugin-mapping.properties b/plugin-mapping.properties index da3890a70a0..bd38586a959 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -96,6 +96,7 @@ seatunnel.sink.Kafka = connector-kafka seatunnel.source.Http = connector-http seatunnel.source.Socket = connector-socket seatunnel.sink.Hive = connector-hive +seatunnel.source.Hive = connector-hive seatunnel.source.Clickhouse = connector-clickhouse seatunnel.sink.Clickhouse = connector-clickhouse seatunnel.sink.ClickhouseFile = connector-clickhouse diff --git a/pom.xml b/pom.xml index 23e15e3c4b3..9339d5f40fb 100644 --- a/pom.xml +++ b/pom.xml @@ -77,6 +77,7 @@ you need to open this annotation and change the dependency of config-shade to project. seatunnel-config --> + seatunnel-dependency-shade seatunnel-common seatunnel-apis seatunnel-core @@ -108,6 +109,17 @@ 0.13.1 1.13.6 0.10.0 + 1.5.6 + 2.3.9 + 1.2 + 1.9.3 + 1.7 + 1.8.1 + 4.3.0 + 1.9.13 + 3.1.6 + 1.19 + 2.1 2.7 2.12.6 1.18.0 @@ -427,7 +439,7 @@ spark-streaming-kafka-0-10_${scala.binary.version} ${spark.version} - + org.projectlombok lombok @@ -663,6 +675,171 @@ ${snappy-java.version} + + org.apache.orc + orc-core + ${orc.version} + + + javax.servlet + servlet-api + + + + org.apache.logging.log4j + * + + + com.fasterxml.jackson.core + * + + + org.apapche.hadoop + * + + + org.apache.curator + * + + + + + org.codehaus.jackson + jackson-core-asl + ${codehaus.jackson.version} + + + org.codehaus.jackson + jackson-xc + ${codehaus.jackson.version} + + + org.codehaus.jackson + jackson-mapper- + ${codehaus.jackson.version} + + + org.codehaus.jackson + jackson-jaxrs + ${codehaus.jackson.version} + + + org.codehaus.janino + janino + ${janino.version} + + + com.sun.jersey + jersey-json + ${jersey.version} + + + org.apache.hive + hive-exec + ${hive.exec.version} + + + org.pentaho + pentaho-aggdesigner-algorithm + + + javax.servlet + servlet-api + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-web + + + com.fasterxml.jackson.core + * + + + org.apapche.hadoop + * + + + com.github.joshelser + dropwizard-metrics-hadoop-metrics2-reporter + + + org.apache.logging.log4j + * + + + org.apache.zookeeper + zookeeper + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + org.apache.hadoop + hadoop-hdfs + + + + + commons-logging + commons-logging + ${commons.logging.version} + + + commons-beanutils + commons-beanutils + ${commons.beanutils.version} + + + commons-configuration + commons-configuration + ${commons.configuration.version} + + + commons-digester + commons-digester + ${commons.digester.version} + + + org.apache.curator + curator-client + ${curator.version} + + + org.apache.curator + curator-framework + ${curator.version} + + + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.curator + curator-recipes + ${curator.version} + + + com.sun.jersey + jersey-core + ${jersey.version} + + + com.sun.jersey + jersey-server + ${jersey.version} + + + javax.servlet.jsp + jsp-api + ${javax.servlet.jap.version} + + @@ -907,7 +1084,7 @@ - + diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PrepareFailException.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PrepareFailException.java index bb43d07ff11..7cb75b98fed 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PrepareFailException.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/PrepareFailException.java @@ -30,4 +30,9 @@ public PrepareFailException(String pluginName, PluginType type, String message) super(String.format("PluginName: %s, PluginType: %s, Message: %s", pluginName, type.getType(), message)); } + + public PrepareFailException(String pluginName, PluginType type, String message, Throwable cause) { + super(String.format("PluginName: %s, PluginType: %s, Message: %s", pluginName, type.getType(), + message), cause); + } } diff --git a/seatunnel-connectors-v2/connector-hive/pom.xml b/seatunnel-connectors-v2/connector-hive/pom.xml index eb2ffa797de..db75d801eca 100644 --- a/seatunnel-connectors-v2/connector-hive/pom.xml +++ b/seatunnel-connectors-v2/connector-hive/pom.xml @@ -30,6 +30,12 @@ connector-hive + + org.apache.seatunnel + seatunnel-hive-shade + ${project.version} + + org.apache.seatunnel seatunnel-api @@ -48,6 +54,25 @@ commons-lang3 + + org.apache.orc + orc-core + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-hdfs + + + + + org.apache.commons + commons-collections4 + + junit junit diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/config/SourceConfig.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/config/SourceConfig.java new file mode 100644 index 00000000000..7054403164b --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/config/SourceConfig.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.config; + +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; + +public class SourceConfig { + + public static final String FILE_TYPE = "file.type"; + + public static final String DEFAULT_FS = FS_DEFAULT_NAME_KEY; + + public static final String FILE_PATH = "file.path"; +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/exception/HivePluginException.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/exception/HivePluginException.java new file mode 100644 index 00000000000..8e1c22548e3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/exception/HivePluginException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.exception; + +public class HivePluginException extends Exception{ + + public HivePluginException(String message) { + super(message); + } + + public HivePluginException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkConfig.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkConfig.java index 2e08862f7b2..e4495121b48 100644 --- a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkConfig.java +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSinkConfig.java @@ -23,7 +23,7 @@ import lombok.Data; import lombok.NonNull; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.List; diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HadoopConf.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HadoopConf.java new file mode 100644 index 00000000000..31dddf7b49a --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HadoopConf.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class HadoopConf implements Serializable { + + private String hdfsNameKey; + + private String fsHdfsImpl = "org.apache.hadoop.hdfs.DistributedFileSystem"; + + public HadoopConf(String hdfsNameKey) { + this.hdfsNameKey = hdfsNameKey; + } +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSource.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSource.java new file mode 100644 index 00000000000..769c1d768a1 --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSource.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source; + +import static org.apache.seatunnel.connectors.seatunnel.hive.config.SourceConfig.FILE_PATH; +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; + +import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.common.SeaTunnelContext; +import org.apache.seatunnel.api.serialization.DefaultSerializer; +import org.apache.seatunnel.api.serialization.Serializer; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.common.constants.PluginType; +import org.apache.seatunnel.connectors.seatunnel.hive.config.SourceConfig; +import org.apache.seatunnel.connectors.seatunnel.hive.exception.HivePluginException; +import org.apache.seatunnel.connectors.seatunnel.hive.source.file.reader.format.OrcReadStrategy; +import org.apache.seatunnel.connectors.seatunnel.hive.source.file.reader.format.ReadStrategy; +import org.apache.seatunnel.connectors.seatunnel.hive.source.file.reader.format.TextReadStrategy; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import com.google.auto.service.AutoService; + +import java.io.IOException; +import java.util.List; + +@AutoService(SeaTunnelSource.class) +public class HiveSource implements SeaTunnelSource { + + private SeaTunnelContext seaTunnelContext; + + private SeaTunnelRowType typeInfo; + + private ReadStrategy readStrategy; + + private HadoopConf hadoopConf; + + private List filesPath; + + @Override + public String getPluginName() { + return "Hive"; + } + + @Override + public void prepare(Config pluginConfig) { + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, FILE_PATH, FS_DEFAULT_NAME_KEY); + if (!result.isSuccess()) { + throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); + } + // default filetype is text + if ("orc".equals(pluginConfig.getString(SourceConfig.FILE_TYPE))) { + readStrategy = new OrcReadStrategy(); + } else { + readStrategy = new TextReadStrategy(); + } + String path = pluginConfig.getString(FILE_PATH); + hadoopConf = new HadoopConf(pluginConfig.getString(FS_DEFAULT_NAME_KEY)); + try { + filesPath = readStrategy.getFileNamesByPath(hadoopConf, path); + } catch (IOException e) { + throw new PrepareFailException(getPluginName(), PluginType.SOURCE, "Check file path fail."); + } + try { + // should read from config or read from hive metadata( wait catlog done) + this.typeInfo = readStrategy.getSeaTunnelRowTypeInfo(hadoopConf, filesPath.get(0)); + } catch (HivePluginException e) { + throw new PrepareFailException(getPluginName(), PluginType.SOURCE, "Read hive file type error.", e); + } + } + + @Override + public void setSeaTunnelContext(SeaTunnelContext seaTunnelContext) { + this.seaTunnelContext = seaTunnelContext; + } + + @Override + public SeaTunnelDataType getProducedType() { + return this.typeInfo; + } + + @Override + public SourceReader createReader(SourceReader.Context readerContext) throws Exception { + return new HiveSourceReader(this.readStrategy, this.hadoopConf, readerContext); + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.BOUNDED; + } + + @Override + public SourceSplitEnumerator createEnumerator(SourceSplitEnumerator.Context enumeratorContext) throws Exception { + return new HiveSourceSplitEnumerator(enumeratorContext, filesPath); + } + + @Override + public SourceSplitEnumerator restoreEnumerator(SourceSplitEnumerator.Context enumeratorContext, HiveSourceState checkpointState) throws Exception { + return new HiveSourceSplitEnumerator(enumeratorContext, filesPath, checkpointState); + } + + @Override + public Serializer getEnumeratorStateSerializer() { + return new DefaultSerializer<>(); + } +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceReader.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceReader.java new file mode 100644 index 00000000000..570f48e35da --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceReader.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source; + +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.connectors.seatunnel.hive.source.file.reader.format.ReadStrategy; + +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@Slf4j +public class HiveSourceReader implements SourceReader { + + private static final long THREAD_WAIT_TIME = 500L; + + private ReadStrategy readStrategy; + + private HadoopConf hadoopConf; + + private Set sourceSplits; + + private final SourceReader.Context context; + + public HiveSourceReader(ReadStrategy readStrategy, HadoopConf hadoopConf, SourceReader.Context context) { + this.readStrategy = readStrategy; + this.hadoopConf = hadoopConf; + this.context = context; + this.sourceSplits = new HashSet<>(); + } + + @Override + public void open() { + readStrategy.init(hadoopConf); + } + + @Override + public void close() { + + } + + @Override + public void pollNext(Collector output) throws Exception { + if (sourceSplits.isEmpty()) { + Thread.sleep(THREAD_WAIT_TIME); + return; + } + sourceSplits.forEach(source -> { + try { + readStrategy.read(source.splitId(), output); + } catch (Exception e) { + throw new RuntimeException("Hive source read error", e); + } + + }); + context.signalNoMoreElement(); + } + + @Override + public List snapshotState(long checkpointId) { + return new ArrayList<>(sourceSplits); + } + + @Override + public void addSplits(List splits) { + sourceSplits.addAll(splits); + } + + @Override + public void handleNoMoreSplits() { + + } + + @Override + public void notifyCheckpointComplete(long checkpointId) { + + } +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceSplit.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceSplit.java new file mode 100644 index 00000000000..44e062e54c3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceSplit.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source; + +import org.apache.seatunnel.api.source.SourceSplit; + +public class HiveSourceSplit implements SourceSplit { + + private static final long serialVersionUID = -1L; + + private String splitId; + + public HiveSourceSplit(String splitId) { + this.splitId = splitId; + } + + @Override + public String splitId() { + return this.splitId; + } +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceSplitEnumerator.java new file mode 100644 index 00000000000..301b1506fe6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceSplitEnumerator.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source; + +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.common.config.Common; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class HiveSourceSplitEnumerator implements SourceSplitEnumerator { + + private final Context context; + private Set pendingSplit; + private Set assignedSplit; + private List filePaths; + + public HiveSourceSplitEnumerator(SourceSplitEnumerator.Context context, List filePaths) { + this.context = context; + this.filePaths = filePaths; + } + + public HiveSourceSplitEnumerator(SourceSplitEnumerator.Context context, List filePaths, + HiveSourceState sourceState) { + this(context, filePaths); + this.assignedSplit = sourceState.getAssignedSplit(); + } + + @Override + public void open() { + this.assignedSplit = new HashSet<>(); + this.pendingSplit = new HashSet<>(); + } + + @Override + public void run() { + pendingSplit = getHiveFileSplit(); + assignSplit(context.registeredReaders()); + } + + private Set getHiveFileSplit() { + Set hiveSourceSplits = new HashSet<>(); + filePaths.forEach(k -> hiveSourceSplits.add(new HiveSourceSplit(k))); + return hiveSourceSplits; + + } + + @Override + public void close() throws IOException { + + } + + @Override + public void addSplitsBack(List splits, int subtaskId) { + if (!splits.isEmpty()) { + pendingSplit.addAll(splits); + assignSplit(Collections.singletonList(subtaskId)); + } + } + + private void assignSplit(Collection taskIDList) { + Map> readySplit = new HashMap<>(Common.COLLECTION_SIZE); + for (int taskID : taskIDList) { + readySplit.computeIfAbsent(taskID, id -> new ArrayList<>()); + } + + pendingSplit.forEach(s -> readySplit.get(getSplitOwner(s.splitId(), taskIDList.size())) + .add(s)); + readySplit.forEach(context::assignSplit); + assignedSplit.addAll(pendingSplit); + pendingSplit.clear(); + } + + private static int getSplitOwner(String tp, int numReaders) { + return tp.hashCode() % numReaders; + } + + @Override + public int currentUnassignedSplitSize() { + return pendingSplit.size(); + } + + @Override + public void registerReader(int subtaskId) { + if (!pendingSplit.isEmpty()) { + assignSplit(Collections.singletonList(subtaskId)); + } + } + + @Override + public HiveSourceState snapshotState(long checkpointId) { + return new HiveSourceState(assignedSplit); + } + + @Override + public void notifyCheckpointComplete(long checkpointId) { + + } + + @Override + public void handleSplitRequest(int subtaskId) { + + } +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceState.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceState.java new file mode 100644 index 00000000000..f982a71cb22 --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/HiveSourceState.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source; + +import java.io.Serializable; +import java.util.Set; + +public class HiveSourceState implements Serializable { + + + private Set assignedSplit; + + public HiveSourceState(Set assignedSplit) { + this.assignedSplit = assignedSplit; + } + + public Set getAssignedSplit() { + return assignedSplit; + } +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/AbstractReadStrategy.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/AbstractReadStrategy.java new file mode 100644 index 00000000000..50163235c73 --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/AbstractReadStrategy.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source.file.reader.format; + +import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; + +import org.apache.seatunnel.connectors.seatunnel.hive.exception.HivePluginException; +import org.apache.seatunnel.connectors.seatunnel.hive.source.HadoopConf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractReadStrategy implements ReadStrategy { + + HadoopConf hadoopConf; + + @Override + public void init(HadoopConf conf) { + this.hadoopConf = conf; + } + + @Override + public Configuration getConfiguration(HadoopConf hadoopConf) { + Configuration configuration = new Configuration(); + configuration.set(FS_DEFAULT_NAME_KEY, hadoopConf.getHdfsNameKey()); + configuration.set("fs.hdfs.impl", hadoopConf.getFsHdfsImpl()); + return configuration; + } + + Configuration getConfiguration() throws HivePluginException { + if (null == hadoopConf) { + throw new HivePluginException("Not init read config"); + } + return getConfiguration(hadoopConf); + } + + boolean checkFileType(String path) { + return true; + } + + @Override + public List getFileNamesByPath(HadoopConf hadoopConf, String path) throws IOException { + Configuration configuration = getConfiguration(hadoopConf); + List fileNames = new ArrayList<>(); + FileSystem hdfs = FileSystem.get(configuration); + Path listFiles = new Path(path); + FileStatus[] stats = hdfs.listStatus(listFiles); + for (FileStatus fileStatus : stats) { + if (fileStatus.isDirectory()) { + fileNames.addAll(getFileNamesByPath(hadoopConf, fileStatus.getPath().toString())); + continue; + } + if (fileStatus.isFile()) { + + fileNames.add(fileStatus.getPath().toString()); + } + } + return fileNames; + } + +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/OrcReadStrategy.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/OrcReadStrategy.java new file mode 100644 index 00000000000..325c1aaccb9 --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/OrcReadStrategy.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source.file.reader.format; + +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.hive.exception.HivePluginException; +import org.apache.seatunnel.connectors.seatunnel.hive.source.HadoopConf; + +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.orc.OrcFile; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; +import org.apache.hadoop.hive.ql.io.orc.OrcStruct; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.apache.orc.Reader; +import org.apache.orc.TypeDescription; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +@Slf4j +public class OrcReadStrategy extends AbstractReadStrategy { + + private SeaTunnelRowType seaTunnelRowTypeInfo; + + @Override + public void read(String path, Collector output) throws Exception { + if (Boolean.FALSE.equals(checkFileType(path))) { + throw new Exception("please check file type"); + } + JobConf conf = new JobConf(); + Path filePath = new Path(path); + Properties p = new Properties(); + OrcSerde serde = new OrcSerde(); + String columns = String.join(",", seaTunnelRowTypeInfo.getFieldNames()); + p.setProperty("columns", columns); + //support types + serde.initialize(conf, p); + StructObjectInspector inspector = (StructObjectInspector) serde.getObjectInspector(); + InputFormat in = new OrcInputFormat(); + FileInputFormat.setInputPaths(conf, filePath); + InputSplit[] splits = in.getSplits(conf, 1); + + conf.set("hive.io.file.readcolumn.ids", "1"); + RecordReader reader = in.getRecordReader(splits[0], conf, Reporter.NULL); + NullWritable key = reader.createKey(); + OrcStruct value = reader.createValue(); + List fields = inspector.getAllStructFieldRefs(); + while (reader.next(key, value)) { + Object[] datas = new Object[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + Object data = inspector.getStructFieldData(value, fields.get(i)); + if (null != data) { + datas[i] = String.valueOf(data); + } else { + datas[i] = null; + } + } + output.collect(new SeaTunnelRow(datas)); + } + reader.close(); + } + + @Override + public SeaTunnelRowType getSeaTunnelRowTypeInfo(HadoopConf hadoopConf, String path) throws HivePluginException { + + if (null != seaTunnelRowTypeInfo) { + return seaTunnelRowTypeInfo; + } + Configuration configuration = getConfiguration(hadoopConf); + Path dstDir = new Path(path); + Reader reader; + try { + reader = OrcFile.createReader(FileSystem.get(configuration), dstDir); + } catch (IOException e) { + throw new HivePluginException("Create OrcReader Fail", e); + } + + TypeDescription schema = reader.getSchema(); + String[] fields = new String[schema.getFieldNames().size()]; + SeaTunnelDataType[] types = new SeaTunnelDataType[schema.getFieldNames().size()]; + + for (int i = 0; i < schema.getFieldNames().size(); i++) { + fields[i] = schema.getFieldNames().get(i); + types[i] = BasicType.STRING_TYPE; + } + seaTunnelRowTypeInfo = new SeaTunnelRowType(fields, types); + return seaTunnelRowTypeInfo; + } + +} + diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/ReadStrategy.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/ReadStrategy.java new file mode 100644 index 00000000000..5e73019147a --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/ReadStrategy.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source.file.reader.format; + +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.hive.exception.HivePluginException; +import org.apache.seatunnel.connectors.seatunnel.hive.source.HadoopConf; + +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; + +public interface ReadStrategy extends Serializable { + void init(HadoopConf conf); + + Configuration getConfiguration(HadoopConf conf); + + void read(String path, Collector output) throws Exception; + + SeaTunnelRowType getSeaTunnelRowTypeInfo(HadoopConf hadoopConf, String path) throws HivePluginException; + + List getFileNamesByPath(HadoopConf hadoopConf, String path) throws IOException; +} diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/TextReadStrategy.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/TextReadStrategy.java new file mode 100644 index 00000000000..aa899cd510a --- /dev/null +++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/source/file/reader/format/TextReadStrategy.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.hive.source.file.reader.format; + +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.hive.exception.HivePluginException; +import org.apache.seatunnel.connectors.seatunnel.hive.source.HadoopConf; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +public class TextReadStrategy extends AbstractReadStrategy { + + private static final String TEXT_FIELD_NAME = "lines"; + + @Override + public void read(String path, Collector output) throws IOException, HivePluginException { + Configuration conf = getConfiguration(); + FileSystem fs = FileSystem.get(conf); + Path filePath = new Path(path); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(filePath), StandardCharsets.UTF_8))) { + reader.lines().forEach(line -> output.collect(new SeaTunnelRow(new String[]{"TEXT_FIELD_NAME", line}))); + } + } + + @Override + public SeaTunnelRowType getSeaTunnelRowTypeInfo(HadoopConf hadoopConf, String path) { + return new SeaTunnelRowType(new String[]{TEXT_FIELD_NAME}, + new SeaTunnelDataType[]{BasicType.STRING_TYPE}); + } +} diff --git a/seatunnel-dependency-shade/pom.xml b/seatunnel-dependency-shade/pom.xml new file mode 100644 index 00000000000..1e6259991fa --- /dev/null +++ b/seatunnel-dependency-shade/pom.xml @@ -0,0 +1,39 @@ + + + + + + seatunnel + org.apache.seatunnel + ${revision} + ../pom.xml + + 4.0.0 + + seatunnel-dependency-shade + pom + + + seatunnel-hive-shade + + + \ No newline at end of file diff --git a/seatunnel-dependency-shade/seatunnel-hive-shade/pom.xml b/seatunnel-dependency-shade/seatunnel-hive-shade/pom.xml new file mode 100644 index 00000000000..ddab451b6b9 --- /dev/null +++ b/seatunnel-dependency-shade/seatunnel-hive-shade/pom.xml @@ -0,0 +1,69 @@ + + + + + seatunnel-dependency-shade + org.apache.seatunnel + ${revision} + + 4.0.0 + seatunnel-hive-shade + + + + org.apache.hive + hive-exec + + + + ${project.artifactId}-${project.version} + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + org/apache/calcite/** + org/htrace/** + + + + + + + + + + + \ No newline at end of file diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE index 4e4d2f912e7..2a9a3c9afa7 100644 --- a/seatunnel-dist/release-docs/LICENSE +++ b/seatunnel-dist/release-docs/LICENSE @@ -556,7 +556,23 @@ The text of each license is the standard Apache 2.0 license. (Apache License, Version 2.0) fastutil (it.unimi.dsi:fastutil:6.5.6 - http://fasutil.dsi.unimi.it/) (Apache License, Version 2.0) fastutil (it.unimi.dsi:fastutil:7.0.13 - http://fasutil.di.unimi.it/) (Apache License, Version 2.0) fastutil (it.unimi.dsi:fastutil:8.5.4 - http://fastutil.di.unimi.it/) + (Apache License, Version 2.0) hadoop-mapreduce-client-app (org.apache.hadoop:hadoop-mapreduce-client-app:2.6.5 - no url defined) + (Apache License, Version 2.0) hadoop-mapreduce-client-common (org.apache.hadoop:hadoop-mapreduce-client-common:2.6.5 - no url defined) + (Apache License, Version 2.0) hadoop-mapreduce-client-common (org.apache.hadoop:hadoop-mapreduce-client-common:3.0.0 - no url defined) + (Apache License, Version 2.0) hadoop-mapreduce-client-core (org.apache.hadoop:hadoop-mapreduce-client-core:2.6.5 - no url defined) (Apache License, Version 2.0) hadoop-mapreduce-client-core (org.apache.hadoop:hadoop-mapreduce-client-core:2.7.7 - no url defined) + (Apache License, Version 2.0) hadoop-mapreduce-client-core (org.apache.hadoop:hadoop-mapreduce-client-core:3.0.0 - no url defined) + (Apache License, Version 2.0) hadoop-mapreduce-client-jobclient (org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.6.5 - no url defined) + (Apache License, Version 2.0) hadoop-mapreduce-client-jobclient (org.apache.hadoop:hadoop-mapreduce-client-jobclient:3.0.0 - no url defined) + (Apache License, Version 2.0) hadoop-mapreduce-client-shuffle (org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.6.5 - no url defined) + (Apache License, Version 2.0) hadoop-yarn-api (org.apache.hadoop:hadoop-yarn-api:2.6.5 - no url defined) + (Apache License, Version 2.0) hadoop-yarn-api (org.apache.hadoop:hadoop-yarn-api:3.0.0 - no url defined) + (Apache License, Version 2.0) hadoop-yarn-client (org.apache.hadoop:hadoop-yarn-client:2.6.5 - no url defined) + (Apache License, Version 2.0) hadoop-yarn-client (org.apache.hadoop:hadoop-yarn-client:3.0.0 - no url defined) + (Apache License, Version 2.0) hadoop-yarn-common (org.apache.hadoop:hadoop-yarn-common:2.6.5 - no url defined) + (Apache License, Version 2.0) hadoop-yarn-common (org.apache.hadoop:hadoop-yarn-common:2.7.7 - no url defined) + (Apache License, Version 2.0) hadoop-yarn-common (org.apache.hadoop:hadoop-yarn-common:3.0.0 - no url defined) + (Apache License, Version 2.0) hadoop-yarn-server-common (org.apache.hadoop:hadoop-yarn-server-common:2.6.5 - no url defined) (Apache License, Version 2.0) htrace-core4 (org.apache.htrace:htrace-core4:4.2.0-incubating - http://incubator.apache.org/projects/htrace.html) (Apache License, Version 2.0) hudi-spark-bundle_2.11 (org.apache.hudi:hudi-spark-bundle_2.11:0.10.0 - https://github.com/apache/hudi/hudi-spark-bundle_2.11) (Apache License, Version 2.0) java-xmlbuilder (com.jamesmurty.utils:java-xmlbuilder:0.4 - http://code.google.com/p/java-xmlbuilder/) @@ -664,24 +680,14 @@ The text of each license is the standard Apache 2.0 license. (The Apache Software License, Version 2.0) Commons Digester (commons-digester:commons-digester:1.8.1 - http://commons.apache.org/digester/) (The Apache Software License, Version 2.0) Commons IO (commons-io:commons-io:2.4 - http://commons.apache.org/io/) (The Apache Software License, Version 2.0) Commons Lang (commons-lang:commons-lang:2.6 - http://commons.apache.org/lang/) - (The Apache Software License, Version 2.0) Commons Logging (commons-logging:commons-logging:1.1.3 - http://commons.apache.org/proper/commons-logging/) (The Apache Software License, Version 2.0) Commons Math (org.apache.commons:commons-math3:3.1.1 - http://commons.apache.org/math/) (The Apache Software License, Version 2.0) Commons Net (commons-net:commons-net:3.1 - http://commons.apache.org/net/) (The Apache Software License, Version 2.0) Commons Pool (commons-pool:commons-pool:1.5.4 - http://commons.apache.org/pool/) (The Apache Software License, Version 2.0) Commons Pool (commons-pool:commons-pool:1.6 - http://commons.apache.org/pool/) (The Apache Software License, Version 2.0) Converter: Moshi (com.squareup.retrofit2:converter-moshi:2.9.0 - https://github.com/square/retrofit) (The Apache Software License, Version 2.0) CronScheduler (io.timeandspace:cron-scheduler:0.1 - https://github.com/TimeAndSpaceIO/CronScheduler) - (The Apache Software License, Version 2.0) Curator Client (org.apache.curator:curator-client:2.12.0 - http://curator.apache.org/curator-client) - (The Apache Software License, Version 2.0) Curator Client (org.apache.curator:curator-client:2.6.0 - http://curator.apache.org/curator-client) - (The Apache Software License, Version 2.0) Curator Client (org.apache.curator:curator-client:2.7.1 - http://curator.apache.org/curator-client) (The Apache Software License, Version 2.0) Curator Client (org.apache.curator:curator-client:4.3.0 - http://curator.apache.org/curator-client) - (The Apache Software License, Version 2.0) Curator Framework (org.apache.curator:curator-framework:2.12.0 - http://curator.apache.org/curator-framework) - (The Apache Software License, Version 2.0) Curator Framework (org.apache.curator:curator-framework:2.6.0 - http://curator.apache.org/curator-framework) - (The Apache Software License, Version 2.0) Curator Framework (org.apache.curator:curator-framework:2.7.1 - http://curator.apache.org/curator-framework) (The Apache Software License, Version 2.0) Curator Framework (org.apache.curator:curator-framework:4.3.0 - http://curator.apache.org/curator-framework) - (The Apache Software License, Version 2.0) Curator Recipes (org.apache.curator:curator-recipes:2.12.0 - http://curator.apache.org/curator-recipes) - (The Apache Software License, Version 2.0) Curator Recipes (org.apache.curator:curator-recipes:2.6.0 - http://curator.apache.org/curator-recipes) - (The Apache Software License, Version 2.0) Curator Recipes (org.apache.curator:curator-recipes:2.7.1 - http://curator.apache.org/curator-recipes) (The Apache Software License, Version 2.0) Curator Recipes (org.apache.curator:curator-recipes:4.3.0 - http://curator.apache.org/curator-recipes) (The Apache Software License, Version 2.0) Curator Service Discovery (org.apache.curator:curator-x-discovery:4.3.0 - http://curator.apache.org/curator-x-discovery) (The Apache Software License, Version 2.0) Data Mapper for Jackson (org.codehaus.jackson:jackson-mapper-asl:1.9.13 - http://jackson.codehaus.org) @@ -756,7 +762,6 @@ The text of each license is the standard Apache 2.0 license. (The Apache Software License, Version 2.0) JPam (net.sf.jpam:jpam:1.1 - http://jpam.sf.net) (The Apache Software License, Version 2.0) JSON Small and Fast Parser (net.minidev:json-smart:2.3 - http://www.minidev.net/) (The Apache Software License, Version 2.0) Jackson (org.codehaus.jackson:jackson-core-asl:1.9.13 - http://jackson.codehaus.org) - (The Apache Software License, Version 2.0) Jackson (org.codehaus.jackson:jackson-core-asl:1.9.2 - http://jackson.codehaus.org) (The Apache Software License, Version 2.0) Jackson 2 extensions to the Google HTTP Client Library for Java. (com.google.http-client:google-http-client-jackson2:1.26.0 - https://github.com/googleapis/google-http-java-client/google-http-client-jackson2) (The Apache Software License, Version 2.0) Jackson dataformat: CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.3 - http://github.com/FasterXML/jackson-dataformats-binary) (The Apache Software License, Version 2.0) Jackson dataformat: CBOR (com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10 - http://github.com/FasterXML/jackson-dataformats-binary) @@ -870,18 +875,58 @@ The text of each license is the standard Apache 2.0 license. (The Apache Software License, Version 2.0) secure-sm (org.elasticsearch:elasticsearch-secure-sm:6.3.1 - https://github.com/elastic/elasticsearch) (The Apache Software License, Version 2.0) server (org.elasticsearch:elasticsearch:6.3.1 - https://github.com/elastic/elasticsearch) (The Apache Software License, Version 2.0) server (org.elasticsearch:elasticsearch:7.5.1 - https://github.com/elastic/elasticsearch) - (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.2.6 - https://github.com/xerial/snappy-java) - (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.4 - https://github.com/xerial/snappy-java) - (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.1 - https://github.com/xerial/snappy-java) - (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.3 - https://github.com/xerial/snappy-java) + (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.8.3 - https://github.com/xerial/snappy-java) (The Apache Software License, Version 2.0) transport (org.elasticsearch.client:transport:6.3.1 - https://github.com/elastic/elasticsearch) (The Apache Software License, Version 2.0) transport (org.elasticsearch.client:transport:7.5.1 - https://github.com/elastic/elasticsearch) (The Apache Software License, Version 2.0) transport-netty4 (org.elasticsearch.plugin:transport-netty4-client:6.3.1 - https://github.com/elastic/elasticsearch) (The Apache Software License, Version 2.0) transport-netty4 (org.elasticsearch.plugin:transport-netty4-client:7.5.1 - https://github.com/elastic/elasticsearch) (The Apache Software License, Version 2.0) x-content (org.elasticsearch:elasticsearch-x-content:6.3.1 - https://github.com/elastic/elasticsearch) (The Apache Software License, Version 2.0) zookeeper (org.apache.zookeeper:zookeeper:3.4.10 - no url defined) - - + (Apache License, Version 2.0) Hadoop Metrics2 Reporter for Dropwizard Metrics (com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter:0.1.2 - https://github.com/joshelser/dropwizard-hadoop-metrics2) + (The Apache Software License, Version 2.0) Open JSON (com.tdunning:json:1.8 - https://github.com/tdunning/open-json) + (The Apache Software License, Version 2.0) EL (commons-el:commons-el:1.0 - http://jakarta.apache.org/commons/el/) + (Apache License 2.0) Metrics Core (io.dropwizard.metrics:metrics-core:3.1.0 - http://metrics.codahale.com/metrics-core/) + (Apache License 2.0) Jackson Integration for Metrics (io.dropwizard.metrics:metrics-json:3.1.0 - http://metrics.codahale.com/metrics-json/) + (Apache License 2.0) JVM Integration for Metrics (io.dropwizard.metrics:metrics-jvm:3.1.0 - http://metrics.codahale.com/metrics-jvm/) + (Apache 2) Joda-Time (joda-time:joda-time:2.8.1 - http://www.joda.org/joda-time/) + (Apache License, Version 2.0) eigenbase-properties (net.hydromatic:eigenbase-properties:1.1.5 - http://github.com/julianhyde/eigenbase-properties) + (Apache 2) opencsv (net.sf.opencsv:opencsv:2.3 - http://opencsv.sf.net) + (The Apache Software License, Version 2.0) Apache Ant Core (org.apache.ant:ant:1.9.1 - http://ant.apache.org/) + (The Apache Software License, Version 2.0) Apache Ant Launcher (org.apache.ant:ant-launcher:1.9.1 - http://ant.apache.org/) + (The Apache Software License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.9 - http://commons.apache.org/proper/commons-compress/) + (The Apache Software License, Version 2.0) Annotation 1.0 (org.apache.geronimo.specs:geronimo-annotation_1.0_spec:1.1.1 - http://geronimo.apache.org/specs/geronimo-annotation_1.0_spec) + (The Apache Software License, Version 2.0) Java Authentication SPI for Containers (org.apache.geronimo.specs:geronimo-jaspic_1.0_spec:1.0 - http://geronimo.apache.org/maven/specs/geronimo-jaspic_1.0_spec/1.0) + (The Apache Software License, Version 2.0) JTA 1.1 (org.apache.geronimo.specs:geronimo-jta_1.1_spec:1.1.1 - http://geronimo.apache.org/specs/geronimo-jta_1.1_spec) + (The Apache Software License, Version 2.0) Hive Common (org.apache.hive:hive-common:2.3.9 - https://hive.apache.org/hive-common) + (The Apache Software License, Version 2.0) Hive Query Language (org.apache.hive:hive-exec:2.3.9 - https://hive.apache.org/hive-exec) + (The Apache Software License, Version 2.0) Hive Llap Client (org.apache.hive:hive-llap-client:2.3.9 - https://hive.apache.org/hive-llap-client) + (The Apache Software License, Version 2.0) Hive Llap Common (org.apache.hive:hive-llap-common:2.3.9 - https://hive.apache.org/hive-llap-common) + (The Apache Software License, Version 2.0) Hive Llap Tez (org.apache.hive:hive-llap-tez:2.3.9 - https://hive.apache.org/hive-llap-tez) + (The Apache Software License, Version 2.0) Hive Serde (org.apache.hive:hive-serde:2.3.9 - https://hive.apache.org/hive-serde) + (The Apache Software License, Version 2.0) Hive Service RPC (org.apache.hive:hive-service-rpc:2.3.9 - https://hive.apache.org/hive-service-rpc) + (The Apache Software License, Version 2.0) Hive Shims (org.apache.hive:hive-shims:2.3.9 - https://hive.apache.org/hive-shims) + (The Apache Software License, Version 2.0) Hive Vector-Code-Gen Utilities (org.apache.hive:hive-vector-code-gen:2.3.9 - https://hive.apache.org/hive-vector-code-gen) + (The Apache Software License, Version 2.0) Hive Shims 0.23 (org.apache.hive.shims:hive-shims-0.23:2.3.9 - https://hive.apache.org/hive-shims-0.23) + (The Apache Software License, Version 2.0) Hive Shims Common (org.apache.hive.shims:hive-shims-common:2.3.9 - https://hive.apache.org/hive-shims-common) + (The Apache Software License, Version 2.0) Hive Shims Scheduler (org.apache.hive.shims:hive-shims-scheduler:2.3.9 - https://hive.apache.org/hive-shims-scheduler) + (The Apache Software License, Version 2.0) Apache Ivy (org.apache.ivy:ivy:2.4.0 - http://ant.apache.org/ivy/) + (The Apache Software License, Version 2.0) Apache Parquet Hadoop Bundle (org.apache.parquet:parquet-hadoop-bundle:1.8.1 - https://parquet.apache.org) + (The Apache Software License, Version 2.0) Apache Thrift (org.apache.thrift:libfb303:0.9.3 - http://thrift.apache.org) + (The Apache Software License, Version 2.0) Apache Thrift (org.apache.thrift:libthrift:0.9.3 - http://thrift.apache.org) + (The Apache Software License, Version 2.0) Apache Velocity (org.apache.velocity:velocity:1.5 - http://velocity.apache.org/engine/releases/velocity-1.5/) + (The Apache Software License, Version 2.0) Apache Groovy (org.codehaus.groovy:groovy-all:2.4.4 - http://groovy-lang.org) + (The Apache Software License, Version 2.0) Data Mapper for Jackson (org.codehaus.jackson:jackson-mapper-asl:1.8.13 - http://jackson.codehaus.org) + (Apache Software License - Version 2.0) Jettison (org.codehaus.jettison:jettison:1.1 - nhttps://mvnrepository.com/artifact/org.codehaus.jettison/jettison) + (The Apache Software License, Version 2.0) DataNucleus Core (org.datanucleus:datanucleus-core:4.1.17 - http://www.datanucleus.org/#/datanucleus-core) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 1.0) Jetty Orbit :: Servlet API (org.eclipse.jetty.orbit:javax.servlet:3.0.0.v201112011016 - http://www.eclipse.org/jetty/jetty-orbit/javax.servlet) + (Apache Software License - Version 2.0) (Eclipse Public License - Version 1.0) Jetty Server (org.mortbay.jetty:jetty:6.1.26 - http://www.eclipse.org/jetty/jetty-parent/project/modules/jetty) + (Apache Software License - Version 2.0) oro (oro:oro:2.0.8 - https://mvnrepository.com/artifact/oro/oro/2.0.8) + (The Apache Software License, Version 2.0) StAX API (stax:stax-api:1.0.1 - http://stax.codehaus.org/) + (The Apache Software License, Version 2.0) jasper-compiler (tomcat:jasper-compiler:5.5.23 - http://tomcat.apache.org/jasper-compiler) + (The Apache Software License, Version 2.0) jasper-runtime (tomcat:jasper-runtime:5.5.23 - http://tomcat.apache.org/jasper-runtime) + (The Apache Software License, Version 2.0) JAX-RS provider for JSON content type (org.codehaus.jackson:jackson-jaxrs:1.9.13 - http://jackson.codehaus.org) + (The Apache Software License, Version 2.0) Xml Compatibility extensions for Jackson (org.codehaus.jackson:jackson-xc:1.9.13 - http://jackson.codehaus.org) + ======================================================================== MIT License ======================================================================== @@ -980,6 +1025,13 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (The BSD License) xmlenc Library (xmlenc:xmlenc:0.52 - http://xmlenc.sourceforge.net) (The New BSD License) Jodd Core (org.jodd:jodd-core:3.5.2 - http://jodd.org) (The New BSD License) Py4J (net.sf.py4j:py4j:0.10.7 - http://nexus.sonatype.org/oss-repository-hosting.html/py4j) + (The BSD License) xmlenc Library (xmlenc:xmlenc:0.52 - http://xmlenc.sourceforge.net) + (The BSD License) ASM Core (asm:asm:3.1 - http://asm.objectweb.org/asm/) + (The BSD License) ASM Commons (asm:asm-commons:3.1 - http://asm.objectweb.org/asm-commons/) + (The BSD License) ASM Tree (asm:asm-tree:3.1 - http://asm.objectweb.org/asm-tree/) + (New BSD License) Commons Compiler (org.codehaus.janino:commons-compiler:2.7.6 - http://docs.codehaus.org/display/JANINO/Home/commons-compiler) + (New BSD License) Janino (org.codehaus.janino:janino:3.1.6 - http://docs.codehaus.org/display/JANINO/Home/janino) + (BSD licence) ANTLR ST4 4.0.4 (org.antlr:ST4:4.0.4 - http://www.stringtemplate.org) ======================================================================== @@ -1038,6 +1090,9 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (CDDL License) JavaBeans Activation Framework (com.sun.activation:javax.activation:1.2.0 - http://java.net/all/javax.activation/) (CDDL License) JavaBeans Activation Framework API jar (javax.activation:javax.activation-api:1.2.0 - http://java.net/all/javax.activation-api/) (CDDL License) JavaMail API (com.sun.mail:javax.mail:1.5.6 - http://javamail.java.net/javax.mail) + (Common Development and Distribution License (CDDL) v1.0) JavaMail API (javax.mail:mail:1.4.1 - https://glassfish.dev.java.net/javaee5/mail/) + (COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0) (GNU General Public Library) Streaming API for XML (javax.xml.stream:stax-api:1.0-2 - no url defined) + ======================================================================== diff --git a/seatunnel-dist/release-docs/NOTICE b/seatunnel-dist/release-docs/NOTICE index fe3c949f619..564463f1ac3 100644 --- a/seatunnel-dist/release-docs/NOTICE +++ b/seatunnel-dist/release-docs/NOTICE @@ -4389,4 +4389,144 @@ Copyright 1999-2020 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). +========================================================================= + +Open Json NOTICE + +========================================================================= +Android JSON library +Copyright (C) 2010 The Android Open Source Project + +This product includes software developed by +The Android Open Source Project +========================================================================= + +Metrics NOTICE + +========================================================================= + +Metrics +Copyright 2010-2013 Coda Hale and Yammer, Inc. + +This product includes software developed by Coda Hale and Yammer, Inc. + +This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, +LongAdder), which was released with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ + + +========================================================================= + +Joda-Time NOTICE + +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (https://www.joda.org/). + + +========================================================================= + +eigenbase-properties NOTICE + +========================================================================= + +eigenbase-properties +Copyright (C) 2012-2015, Julian Hyde + +This product includes software from the Eigenbase project, licensed from +DynamoBI Corporation. + +Copyright (C) 2005 Dynamo BI Corporation + +=============================================================================== +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this software except in compliance with the License. +You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific lang + +========================================================================= + + Apache Ant NOTICE + +========================================================================= + Apache Ant + Copyright 1999-2013 The Apache Software Foundation + + The task is based on code Copyright (c) 2002, Landmark + Graphics Corp that has been kindly donated to the Apache Software + Foundation. + + +========================================================================= + +Apache Geronimo NOTICE + +========================================================================= +Apache Geronimo +Copyright 2003-2008 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +========================================================================= + +Apache Ivy NOTICE + +========================================================================= +Apache Ivy (TM) +Copyright 2007-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Portions of Ivy were originally developed at +Jayasoft SARL (http://www.jayasoft.fr/) +and are licensed to the Apache Software Foundation under the +"Software Grant License Agreement" + +SSH and SFTP support is provided by the JCraft JSch package, +which is open source software, available under +the terms of a BSD style license. +The original software and related information is available +at http://www.jcraft.com/jsch/. + +========================================================================= + +Apache Parquet Hadoop Bundle NOTICE + +========================================================================= +Apache Parquet Hadoop Bundle +Copyright 2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). +========================================================================= + +Apache Velocity NOTICE + +========================================================================= + +Apache Velocity + +Copyright (C) 2000-2007 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The files + - velocity-engine-scripting/src/main/java/org/apache/velocity/script/VelocityScriptEngine.java + - velocity-engine-scripting/src/main/java/org/apache/velocity/script/VelocityScriptEngineFactory.java +are Copyright 2006 Sun Microsystems, Inc., and licenced under a BSD-like licence. + ========================================================================= \ No newline at end of file diff --git a/seatunnel-dist/release-docs/licenses/LICENSE-ant.txt b/seatunnel-dist/release-docs/licenses/LICENSE-ant.txt new file mode 100644 index 00000000000..d20c10c913e --- /dev/null +++ b/seatunnel-dist/release-docs/licenses/LICENSE-ant.txt @@ -0,0 +1,48 @@ + + changes were made. (We recommend you provide URIs to the location from + which the code is derived.) + +THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS MAKE +NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT +THE USE OF THE SOFTWARE OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY +PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS. + +COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR DOCUMENTATION. + +The name and trademarks of copyright holders may NOT be used in advertising or +publicity pertaining to the software without specific, written prior permission. +Title to copyright in this software and any associated documentation will at +all times remain with copyright holders. + +____________________________________ + +This formulation of W3C's notice and license became active on December 31 2002. +This version removes the copyright ownership notice such that this license can +be used with materials other than those owned by the W3C, reflects that ERCIM +is now a host of the W3C, includes references to this specific dated version of +the license, and removes the ambiguous grant of "use". Otherwise, this version +is the same as the previous version and is written so as to preserve the Free +Software Foundation's assessment of GPL compatibility and OSI's certification +under the Open Source Definition. Please see our Copyright FAQ for common +questions about using materials from our site, including specific terms and +conditions for packages like libwww, Amaya, and Jigsaw. Other questions about +this notice can be directed to site-policy@w3.org. + +Joseph Reagle + +This license came from: http://www.megginson.com/SAX/copying.html + However please note future versions of SAX may be covered + under http://saxproject.org/?selected=pd + +SAX2 is Free! + +I hereby abandon any property rights to SAX 2.0 (the Simple API for +XML), and release all of the SAX 2.0 source code, compiled code, and +documentation contained in this distribution into the Public Domain. +SAX comes with NO WARRANTY or guarantee of fitness for any +purpose. + +David Megginson, david@megginson.com +2000-05-05 \ No newline at end of file diff --git a/seatunnel-dist/release-docs/licenses/LICENSE-ivy.txt b/seatunnel-dist/release-docs/licenses/LICENSE-ivy.txt new file mode 100644 index 00000000000..87a7f01361d --- /dev/null +++ b/seatunnel-dist/release-docs/licenses/LICENSE-ivy.txt @@ -0,0 +1,259 @@ + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------------------------------------------------------------------ +License for JCraft JSch package +------------------------------------------------------------------------------ +Copyright (c) 2002,2003,2004,2005,2006,2007 Atsuhiko Yamanaka, JCraft,Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + 3. The names of the authors may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, +INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +------------------------------------------------------------------------------ +License for jQuery +------------------------------------------------------------------------------ +Copyright (c) 2007 John Resig, http://jquery.com/ + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/seatunnel-dist/release-docs/licenses/LICENSE-parquet-hadoop-bundle.txt b/seatunnel-dist/release-docs/licenses/LICENSE-parquet-hadoop-bundle.txt new file mode 100644 index 00000000000..b53f78f5c8a --- /dev/null +++ b/seatunnel-dist/release-docs/licenses/LICENSE-parquet-hadoop-bundle.txt @@ -0,0 +1,248 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +-------------------------------------------------------------------------------- + +This product depends on Apache Thrift and includes it in this binary artifact. + +Copyright: 2006-2010 The Apache Software Foundation. +Home page: https://thrift.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product depends on SLF4J and includes SLF4J in this binary artifact. SLF4J +is a simple logging facade for Java. + +Copyright: 2004-2013 QOS.ch. +Home page: http://www.slf4j.org/ +License: http://slf4j.org/license.html (MIT license) + +The following is the SLF4J license (MIT): + + Copyright (c) 2004-2013 QOS.ch + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's JavaFastPFOR project in this +binary artifact. The "Lemire" bit packing classes produced by parquet-generator +are derived from the JavaFastPFOR project. + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/JavaFastPFOR +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product depends on fastutil and includes it in this binary artifact. +Fastutil provides type-specific collection implementations. + +Copyright: 2002-2014 Sebastiano Vigna +Home page: http://fasutil.di.unimi.it/ +License: http://www.apache.org/licenses/LICENSE-2.0.html + +-------------------------------------------------------------------------------- + +This product depends on Jackson and includes it in this binary artifact. +Jackson is a high-performance JSON processor. + +Copyright: 2007-2015 Tatu Saloranta and other contributors +Home page: http://jackson.codehaus.org/ +License: http://www.apache.org/licenses/LICENSE-2.0.txt + diff --git a/seatunnel-dist/release-docs/licenses/LICENSE-st.txt b/seatunnel-dist/release-docs/licenses/LICENSE-st.txt new file mode 100644 index 00000000000..3a33e34b1df --- /dev/null +++ b/seatunnel-dist/release-docs/licenses/LICENSE-st.txt @@ -0,0 +1,26 @@ +[The "BSD license"] +Copyright (c) 2011-2022 Terence Parr +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 9411a44bde3..22d342d4cb9 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -2,6 +2,7 @@ HdrHistogram-2.1.9.jar RoaringBitmap-0.5.11.jar RoaringBitmap-0.9.0.jar RoaringBitmap-0.9.22.jar +ST4-4.0.4.jar SparseBitSet-1.2.jar accessors-smart-1.2.jar activation-1.1.1.jar @@ -18,12 +19,16 @@ aggs-matrix-stats-client-7.5.1.jar aircompressor-0.10.jar airline-0.7.jar annotations-13.0.jar +ant-1.6.5.jar +ant-1.9.1.jar +ant-launcher-1.9.1.jar antlr-runtime-3.5.2.jar antlr4-runtime-4.5.1.jar antlr4-runtime-4.7.jar aopalliance-1.0.jar aopalliance-repackaged-2.4.0-b34.jar aopalliance-repackaged-2.5.0-b32.jar +apache-curator-2.7.1.pom apacheds-i18n-2.0.0-M15.jar apacheds-kerberos-codec-2.0.0-M15.jar api-asn1-api-1.0.0-M20.jar @@ -34,7 +39,9 @@ asm-5.0.4.jar asm-7.1.jar asm-all-5.0.2.jar asm-analysis-7.1.jar +asm-commons-3.1.jar asm-commons-7.1.jar +asm-tree-3.1.jar asm-tree-7.1.jar async-http-client-2.5.3.jar async-http-client-netty-utils-2.5.3.jar @@ -70,15 +77,14 @@ clickhouse-http-client-0.3.2-patch9-shaded.jar clickhouse-http-client-0.3.2-patch9.jar clickhouse-jdbc-0.2.jar clickhouse-jdbc-0.3.2-patch9.jar -commons-beanutils-1.7.0.jar commons-beanutils-1.9.3.jar -commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar commons-cli-1.3.1.jar commons-cli-1.4.jar commons-codec-1.13.jar commons-collections-3.2.2.jar commons-collections4-4.4.jar +commons-compiler-2.7.6.jar commons-compiler-3.0.9.jar commons-compiler-3.1.6.jar commons-compress-1.18.jar @@ -87,7 +93,7 @@ commons-compress-1.20.jar commons-compress-1.21.jar commons-compress-1.4.1.jar commons-compress-1.8.1.jar -commons-configuration-1.6.jar +commons-compress-1.9.jar commons-configuration-1.7.jar commons-configuration2-2.1.1.jar commons-crypto-1.0.0.jar @@ -95,7 +101,7 @@ commons-csv-1.0.jar commons-daemon-1.0.13.jar commons-dbcp2-2.0.1.jar commons-digester-1.8.1.jar -commons-digester-1.8.jar +commons-el-1.0.jar commons-email-1.5.jar commons-httpclient-3.1.jar commons-io-2.11.0.jar @@ -103,7 +109,6 @@ commons-io-2.4.jar commons-io-2.5.jar commons-lang-2.6.jar commons-lang3-3.4.jar -commons-logging-1.1.3.jar commons-logging-1.2.jar commons-math3-3.1.1.jar commons-math3-3.4.1.jar @@ -121,18 +126,12 @@ config-1.3.3.jar config-magic-0.9.jar converter-moshi-2.9.0.jar cron-scheduler-0.1.jar -curator-client-2.12.0.jar -curator-client-2.6.0.jar -curator-client-2.7.1.jar curator-client-4.3.0.jar -curator-framework-2.12.0.jar -curator-framework-2.6.0.jar curator-framework-4.3.0.jar -curator-recipes-2.12.0.jar -curator-recipes-2.6.0.jar curator-recipes-4.3.0.jar curator-x-discovery-4.3.0.jar curvesapi-1.06.jar +datanucleus-core-4.1.17.jar datasketches-java-2.0.0.jar datasketches-memory-1.3.0.jar derby-10.14.2.0.jar @@ -148,6 +147,7 @@ druid-indexing-hadoop-0.22.1.jar druid-indexing-service-0.22.1.jar druid-processing-0.22.1.jar druid-server-0.22.1.jar +eigenbase-properties-1.1.5.jar elasticsearch-6.3.1.jar elasticsearch-7.5.1.jar elasticsearch-cli-6.3.1.jar @@ -190,10 +190,14 @@ flink-shaded-hadoop-2-2.7.5-7.0.jar flink-statebackend-rocksdb_2.11-1.13.6.jar force-shading-1.13.6.jar frocksdbjni-5.17.2-ververica-2.1.jar +geronimo-annotation_1.0_spec-1.1.1.jar +geronimo-jaspic_1.0_spec-1.0.jar +geronimo-jta_1.1_spec-1.1.1.jar google-api-client-1.26.0.jar google-http-client-1.26.0.jar google-http-client-jackson2-1.26.0.jar google-oauth-client-1.26.0.jar +groovy-all-2.4.4.jar gson-2.2.4.jar gson-2.9.0.jar guava-19.0.jar @@ -272,8 +276,21 @@ hbase-zookeeper-2.1.0.jar hibernate-validator-5.2.5.Final.jar hive-beeline-1.2.1.spark2.jar hive-cli-1.2.1.spark2.jar +hive-common-2.3.9.jar +hive-exec-2.3.9.jar hive-jdbc-1.2.1.spark2.jar +hive-llap-client-2.3.9.jar +hive-llap-common-2.3.9.jar +hive-llap-tez-2.3.9.jar +hive-serde-2.3.9.jar +hive-service-rpc-2.3.9.jar +hive-shims-0.23-2.3.9.jar +hive-shims-2.3.9.jar +hive-shims-common-2.3.9.jar +hive-shims-scheduler-2.3.9.jar +hive-storage-api-2.4.0.jar hive-storage-api-2.6.0.jar +hive-vector-code-gen-2.3.9.jar hk2-api-2.4.0-b34.jar hk2-api-2.5.0-b32.jar hk2-locator-2.4.0-b34.jar @@ -304,11 +321,11 @@ icu4j-charset-60.2.jar icu4j-localespi-60.2.jar influxdb-java-2.22.jar ion-java-1.0.2.jar +ivy-2.4.0.jar j2objc-annotations-1.1.jar jackson-annotations-2.12.6.jar jackson-core-2.12.6.jar jackson-core-asl-1.9.13.jar -jackson-core-asl-1.9.2.jar jackson-databind-2.12.6.jar jackson-dataformat-cbor-2.12.3.jar jackson-dataformat-cbor-2.8.10.jar @@ -322,7 +339,6 @@ jackson-dataformat-yaml-2.8.11.jar jackson-datatype-guava-2.10.5.jar jackson-datatype-joda-2.10.5.jar jackson-jaxrs-1.9.13.jar -jackson-jaxrs-1.9.2.jar jackson-jaxrs-base-2.10.5.jar jackson-jaxrs-base-2.7.8.jar jackson-jaxrs-json-provider-2.10.5.jar @@ -335,15 +351,15 @@ jackson-module-guice-2.10.5.jar jackson-module-jaxb-annotations-2.10.5.jar jackson-module-jaxb-annotations-2.7.8.jar jackson-xc-1.9.13.jar -jackson-xc-1.9.2.jar jakarta.activation-api-1.2.1.jar jakarta.activation-api-1.2.2.jar jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api-2.3.2.jar jakarta.xml.bind-api-2.3.3.jar jamon-runtime-2.4.1.jar -janino-3.0.9.jar janino-3.1.6.jar +jasper-compiler-5.5.23.jar +jasper-runtime-5.5.23.jar java-xmlbuilder-0.4.jar javassist-3.18.1-GA.jar javassist-3.20.0-GA.jar @@ -357,6 +373,7 @@ javax.inject-1.jar javax.inject-2.4.0-b34.jar javax.inject-2.5.0-b32.jar javax.mail-1.5.6.jar +javax.servlet-3.0.0.v201112011016.jar javax.servlet-api-3.1.0.jar javax.servlet.jsp-api-2.3.1.jar javax.ws.rs-api-2.0.1.jar @@ -382,28 +399,22 @@ jersey-client-2.22.2.jar jersey-common-2.22.2.jar jersey-container-servlet-core-2.22.2.jar jersey-container-servlet-core-2.25.1.jar -jersey-core-1.19.3.jar jersey-core-1.19.jar -jersey-core-1.9.jar jersey-guava-2.22.2.jar jersey-guice-1.19.3.jar jersey-guice-1.19.jar jersey-guice-1.9.jar jersey-json-1.19.jar -jersey-json-1.9.jar jersey-media-jaxb-2.22.2.jar jersey-media-jaxb-2.25.1.jar -jersey-server-1.19.3.jar jersey-server-1.19.jar -jersey-server-1.9.jar jersey-server-2.22.2.jar jersey-server-2.25.1.jar jersey-servlet-1.19.3.jar jersey-servlet-1.19.jar -jets3t-0.9.0.jar jettison-1.1.jar jettison-1.3.8.jar -jetty-6.1.26.jar +jetty-all-7.6.0.v20120127.jar jetty-client-9.4.40.v20210413.jar jetty-continuation-9.4.40.v20210413.jar jetty-http-9.3.19.v20170502.jar @@ -419,6 +430,8 @@ jetty-servlet-9.3.19.v20170502.jar jetty-servlet-9.4.40.v20210413.jar jetty-servlets-9.4.40.v20210413.jar jetty-sslengine-6.1.26.jar +jetty-6.1.26.jar +jets3t-0.9.0.jar jetty-util-6.1.26.jar jetty-util-9.3.19.v20170502.jar jetty-util-9.4.40.v20210413.jar @@ -435,6 +448,7 @@ jna-4.5.1.jar joda-time-1.6.jar joda-time-2.10.3.jar joda-time-2.10.5.jar +joda-time-2.8.1.jar joda-time-2.9.9.jar joni-2.1.11.jar joni-2.1.2.jar @@ -442,11 +456,14 @@ joni-2.1.27.jar jopt-simple-5.0.2.jar jpam-1.1.jar jsch-0.1.54.jar +json-1.8.jar json-path-2.3.0.jar json-smart-2.3.jar +jsp-api-2.0.jar jsp-api-2.1.jar jsr305-1.3.9.jar jsr305-2.0.1.jar +jsr305-3.0.0.jar jsr311-api-1.1.1.jar jul-to-slf4j-1.7.25.jar jvm-attach-api-1.5.jar @@ -476,7 +493,9 @@ kudu-spark2_2.11-1.7.0.jar lang-mustache-client-6.3.1.jar lang-mustache-client-7.5.1.jar leveldbjni-all-1.8.jar +libfb303-0.9.3.jar libthrift-0.9.0.jar +libthrift-0.9.3.jar log4j-1.2-api-2.15.0.jar log4j-1.2.14.jar log4j-1.2.17.jar @@ -524,6 +543,7 @@ lz4-java-1.4.1.jar lz4-java-1.6.0.jar lz4-java-1.7.1.jar lz4-java-1.8.0.jar +mail-1.4.1.jar mapper-extras-client-7.5.1.jar maven-aether-provider-3.1.1.jar maven-artifact-3.6.0.jar @@ -533,9 +553,12 @@ maven-repository-metadata-3.1.1.jar maven-settings-3.1.1.jar maven-settings-builder-3.1.1.jar memory-0.9.0.jar +metrics-core-3.1.0.jar metrics-core-3.1.5.jar metrics-core-3.2.1.jar metrics-core-4.0.0.jar +metrics-json-3.1.0.jar +metrics-jvm-3.1.0.jar minlog-1.3.0.jar mongo-java-driver-3.4.2.jar mongo-spark-connector_2.11-2.2.0.jar @@ -585,10 +608,12 @@ okhttp-2.4.0.jar okhttp-4.9.1.jar okio-1.4.0.jar okio-2.8.0.jar +opencsv-2.3.jar opencsv-4.6.jar orc-core-1.5.6.jar orc-shims-1.5.2.jar orc-shims-1.5.6.jar +oro-2.0.8.jar osgi-resource-locator-1.0.1.jar paranamer-2.3.jar paranamer-2.7.jar @@ -605,6 +630,7 @@ parquet-format-2.4.0.jar parquet-format-structures-1.11.1.jar parquet-hadoop-1.10.0.jar parquet-hadoop-1.11.1.jar +parquet-hadoop-bundle-1.8.1.jar parquet-jackson-1.10.0.jar parquet-jackson-1.11.1.jar percolator-client-6.3.1.jar @@ -668,6 +694,7 @@ sqlline-1.2.0.jar sshd-common-2.7.0.jar sshd-core-2.7.0.jar sshd-scp-2.7.0.jar +stax-api-1.0.1.jar stax-api-1.0-2.jar stax2-api-3.1.4.jar stream-2.7.0.jar @@ -697,6 +724,7 @@ uzaygezen-core-0.2.jar validation-api-1.1.0.Final.jar vavr-0.10.2.jar vavr-match-0.10.2.jar +velocity-1.5.jar wagon-provider-api-2.4.jar woodstox-core-5.0.3.jar xbean-asm6-shaded-4.10.jar From a31fdeedcc1629d6c24c3033a57ad397f2a37e71 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Tue, 5 Jul 2022 13:42:54 +0800 Subject: [PATCH 11/76] [SeaTunnel API] [Sink] remove useless context field (#2124) --- docs/en/connector/source/Http.md | 2 +- .../api/sink/DefaultSinkWriterContext.java | 18 +----------------- .../apache/seatunnel/api/sink/SinkWriter.java | 11 ----------- .../jdbc/internal/xa/SemanticXidGenerator.java | 3 +-- .../flink/execution/SinkExecuteProcessor.java | 3 +-- .../spark/execution/SinkExecuteProcessor.java | 5 +---- .../translation/flink/sink/FlinkSink.java | 11 ++--------- .../spark/sink/SparkDataSourceWriter.java | 9 +++------ .../spark/sink/SparkDataWriterFactory.java | 10 ++-------- .../translation/spark/sink/SparkSink.java | 9 ++------- .../spark/sink/SparkSinkInjector.java | 10 ++-------- .../spark/sink/SparkStreamWriter.java | 6 ++---- 12 files changed, 18 insertions(+), 79 deletions(-) diff --git a/docs/en/connector/source/Http.md b/docs/en/connector/source/Http.md index d9d3a399ba3..ad34e9e7469 100644 --- a/docs/en/connector/source/Http.md +++ b/docs/en/connector/source/Http.md @@ -37,7 +37,7 @@ HTTP request header, json format. ### request_params[string] -HTTP request parameters, json format. +HTTP request parameters, json format. Use string with escapes to save json ### sync_path[string] diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSinkWriterContext.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSinkWriterContext.java index 8c8219db71d..c8a3fd61ff9 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSinkWriterContext.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/DefaultSinkWriterContext.java @@ -17,26 +17,14 @@ package org.apache.seatunnel.api.sink; -import java.util.Map; - /** * The default {@link SinkWriter.Context} implement class. */ public class DefaultSinkWriterContext implements SinkWriter.Context { - - private final Map configuration; private final int subtask; - private final int parallelism; - public DefaultSinkWriterContext(Map configuration, int subtask, int parallelism) { - this.configuration = configuration; + public DefaultSinkWriterContext(int subtask) { this.subtask = subtask; - this.parallelism = parallelism; - } - - @Override - public Map getConfiguration() { - return configuration; } @Override @@ -44,8 +32,4 @@ public int getIndexOfSubtask() { return subtask; } - @Override - public int getNumberOfParallelSubtasks() { - return parallelism; - } } diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkWriter.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkWriter.java index 268d3d40e6a..897e64b4cea 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkWriter.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/sink/SinkWriter.java @@ -21,7 +21,6 @@ import java.io.Serializable; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; /** @@ -76,20 +75,10 @@ default List snapshotState(long checkpointId) throws IOException { interface Context extends Serializable{ - /** - * Gets the configuration with which Job was started. - */ - Map getConfiguration(); - /** * @return The index of this subtask. */ int getIndexOfSubtask(); - /** - * @return The number of parallel Sink tasks. - */ - int getNumberOfParallelSubtasks(); - } } diff --git a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/xa/SemanticXidGenerator.java b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/xa/SemanticXidGenerator.java index 98825a57a59..3d2a82b3d13 100644 --- a/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/xa/SemanticXidGenerator.java +++ b/seatunnel-connectors-v2/connector-jdbc/src/main/java/org/apache/seatunnel/connectors/seatunnel/jdbc/internal/xa/SemanticXidGenerator.java @@ -80,8 +80,7 @@ public boolean belongsToSubtask(Xid xid, SeaTunnelContext context, SinkWriter.Co return false; } int subtaskIndex = readNumber(xid.getGlobalTransactionId(), JOB_ID_BYTES, Integer.BYTES); - if (subtaskIndex != sinkContext.getIndexOfSubtask() - && subtaskIndex <= sinkContext.getNumberOfParallelSubtasks() - 1) { + if (subtaskIndex != sinkContext.getIndexOfSubtask()) { return false; } byte[] jobIdBytes = new byte[JOB_ID_BYTES]; diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SinkExecuteProcessor.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SinkExecuteProcessor.java index 079e3f95e72..4744fdf7e94 100644 --- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SinkExecuteProcessor.java +++ b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/SinkExecuteProcessor.java @@ -36,7 +36,6 @@ import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -76,7 +75,7 @@ public List> execute(List> upstreamDataStreams) SeaTunnelSink seaTunnelSink = plugins.get(i); DataStream stream = fromSourceTable(sinkConfig).orElse(input); seaTunnelSink.setTypeInfo((SeaTunnelRowType) TypeConverterUtils.convert(stream.getType())); - stream.sinkTo(new FlinkSink<>(seaTunnelSink, Collections.emptyMap())); + stream.sinkTo(new FlinkSink<>(seaTunnelSink)); } // the sink is the last stream return null; diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java index c19770a6347..2e0096a471e 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SinkExecuteProcessor.java @@ -20,7 +20,6 @@ import org.apache.seatunnel.api.common.SeaTunnelContext; import org.apache.seatunnel.api.sink.SeaTunnelSink; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.config.Common; import org.apache.seatunnel.core.starter.exception.TaskExecuteException; import org.apache.seatunnel.plugin.discovery.PluginIdentifier; import org.apache.seatunnel.plugin.discovery.seatunnel.SeaTunnelSinkPluginDiscovery; @@ -36,7 +35,6 @@ import java.net.URL; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.stream.Collectors; @@ -74,8 +72,7 @@ public List> execute(List> upstreamDataStreams) throws Dataset dataset = fromSourceTable(sinkConfig, sparkEnvironment).orElse(input); // TODO modify checkpoint location seaTunnelSink.setTypeInfo((SeaTunnelRowType) TypeConverterUtils.convert(dataset.schema())); - SparkSinkInjector.inject(dataset.write(), seaTunnelSink, new HashMap<>(Common.COLLECTION_SIZE)).option( - "checkpointLocation", "/tmp").save(); + SparkSinkInjector.inject(dataset.write(), seaTunnelSink).option("checkpointLocation", "/tmp").save(); } // the sink is the last stream return null; diff --git a/seatunnel-translation/seatunnel-translation-flink/src/main/java/org/apache/seatunnel/translation/flink/sink/FlinkSink.java b/seatunnel-translation/seatunnel-translation-flink/src/main/java/org/apache/seatunnel/translation/flink/sink/FlinkSink.java index c90fd86eb75..a3e9272407d 100644 --- a/seatunnel-translation/seatunnel-translation-flink/src/main/java/org/apache/seatunnel/translation/flink/sink/FlinkSink.java +++ b/seatunnel-translation/seatunnel-translation-flink/src/main/java/org/apache/seatunnel/translation/flink/sink/FlinkSink.java @@ -32,28 +32,21 @@ import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -@SuppressWarnings("unchecked") public class FlinkSink implements Sink, FlinkWriterState, GlobalCommT> { private final SeaTunnelSink sink; - private final Map configuration; - public FlinkSink(SeaTunnelSink sink, - Map configuration) { + public FlinkSink(SeaTunnelSink sink) { this.sink = sink; - this.configuration = configuration; } @Override public SinkWriter, FlinkWriterState> createWriter(org.apache.flink.api.connector.sink.Sink.InitContext context, List> states) throws IOException { - // TODO add subtask and parallelism. - org.apache.seatunnel.api.sink.SinkWriter.Context stContext = - new DefaultSinkWriterContext(configuration, 0, 0); + org.apache.seatunnel.api.sink.SinkWriter.Context stContext = new DefaultSinkWriterContext(context.getSubtaskId()); if (states == null || states.isEmpty()) { return new FlinkSinkWriter<>(sink.createWriter(stContext), 1, sink.getConsumedType()); diff --git a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataSourceWriter.java b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataSourceWriter.java index 3d79912d231..4855f706624 100644 --- a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataSourceWriter.java +++ b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataSourceWriter.java @@ -33,27 +33,24 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; public class SparkDataSourceWriter implements DataSourceWriter { protected final SeaTunnelSink sink; - protected final Map configuration; @Nullable protected final SinkAggregatedCommitter sinkAggregatedCommitter; - SparkDataSourceWriter(SeaTunnelSink sink, - Map configuration) throws IOException { + SparkDataSourceWriter(SeaTunnelSink sink) + throws IOException { this.sink = sink; - this.configuration = configuration; this.sinkAggregatedCommitter = sink.createAggregatedCommitter().orElse(null); } @Override public DataWriterFactory createWriterFactory() { - return new SparkDataWriterFactory<>(sink, configuration); + return new SparkDataWriterFactory<>(sink); } @Override diff --git a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataWriterFactory.java b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataWriterFactory.java index 8cdbca3b977..ac3c6aa58e9 100644 --- a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataWriterFactory.java +++ b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkDataWriterFactory.java @@ -28,23 +28,17 @@ import org.apache.spark.sql.sources.v2.writer.DataWriterFactory; import java.io.IOException; -import java.util.Map; public class SparkDataWriterFactory implements DataWriterFactory { private final SeaTunnelSink sink; - private final Map configuration; - SparkDataWriterFactory(SeaTunnelSink sink, Map configuration) { + SparkDataWriterFactory(SeaTunnelSink sink) { this.sink = sink; - this.configuration = configuration; } @Override public DataWriter createDataWriter(int partitionId, long taskId, long epochId) { - // TODO use partitionID, taskId information. - // TODO add subtask and parallelism. - org.apache.seatunnel.api.sink.SinkWriter.Context context = - new DefaultSinkWriterContext(configuration, (int) taskId, 0); + org.apache.seatunnel.api.sink.SinkWriter.Context context = new DefaultSinkWriterContext((int) taskId); SinkWriter writer; SinkCommitter committer; try { diff --git a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java index dd9efe540df..b174fb14405 100644 --- a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java +++ b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSink.java @@ -32,23 +32,18 @@ import org.apache.spark.sql.types.StructType; import java.io.IOException; -import java.util.Map; import java.util.Optional; public class SparkSink implements WriteSupport, StreamWriteSupport, DataSourceV2 { private volatile SeaTunnelSink sink; - private Map configuration; private void init(DataSourceOptions options) { if (sink == null) { this.sink = SerializationUtils.stringToObject( options.get("sink").orElseThrow(() -> new IllegalArgumentException("can not find sink " + "class string in DataSourceOptions"))); - this.configuration = SerializationUtils.stringToObject( - options.get("configuration").orElseThrow(() -> new IllegalArgumentException("can not " + - "find configuration class string in DataSourceOptions"))); } } @@ -57,7 +52,7 @@ public StreamWriter createStreamWriter(String queryId, StructType schema, Output init(options); try { - return new SparkStreamWriter<>(sink, configuration); + return new SparkStreamWriter<>(sink); } catch (IOException e) { throw new RuntimeException("find error when createStreamWriter", e); } @@ -68,7 +63,7 @@ public Optional createWriter(String writeUUID, StructType sche init(options); try { - return Optional.of(new SparkDataSourceWriter<>(sink, configuration)); + return Optional.of(new SparkDataSourceWriter<>(sink)); } catch (IOException e) { throw new RuntimeException("find error when createStreamWriter", e); } diff --git a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java index 62cdc255c79..00d6a5daaf6 100644 --- a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java +++ b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkSinkInjector.java @@ -25,24 +25,18 @@ import org.apache.spark.sql.streaming.DataStreamWriter; import org.apache.spark.sql.streaming.OutputMode; -import java.util.HashMap; - public class SparkSinkInjector { private static final String SPARK_SINK_CLASS_NAME = "org.apache.seatunnel.translation.spark.sink.SparkSink"; - public static DataStreamWriter inject(DataStreamWriter dataset, SeaTunnelSink sink, - HashMap configuration) { + public static DataStreamWriter inject(DataStreamWriter dataset, SeaTunnelSink sink) { return dataset.format(SPARK_SINK_CLASS_NAME) .outputMode(OutputMode.Append()) - .option("configuration", SerializationUtils.objectToString(configuration)) .option("sink", SerializationUtils.objectToString(sink)); } - public static DataFrameWriter inject(DataFrameWriter dataset, SeaTunnelSink sink, - HashMap configuration) { + public static DataFrameWriter inject(DataFrameWriter dataset, SeaTunnelSink sink) { return dataset.format(SPARK_SINK_CLASS_NAME) - .option("configuration", SerializationUtils.objectToString(configuration)) .option("sink", SerializationUtils.objectToString(sink)); } diff --git a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkStreamWriter.java b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkStreamWriter.java index 2fc678e3858..85f6a9542a9 100644 --- a/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkStreamWriter.java +++ b/seatunnel-translation/seatunnel-translation-spark/src/main/java/org/apache/seatunnel/translation/spark/sink/SparkStreamWriter.java @@ -26,14 +26,12 @@ import org.apache.spark.sql.sources.v2.writer.streaming.StreamWriter; import java.io.IOException; -import java.util.Map; public class SparkStreamWriter extends SparkDataSourceWriter implements StreamWriter { - SparkStreamWriter(SeaTunnelSink sink, - Map configuration) throws IOException { - super(sink, configuration); + SparkStreamWriter(SeaTunnelSink sink) throws IOException { + super(sink); } @Override From e807b3402a5924ccc97d46615ae1d30ac9169e8e Mon Sep 17 00:00:00 2001 From: Kirs Date: Tue, 5 Jul 2022 15:36:59 +0800 Subject: [PATCH 12/76] [Bump]Upgrade commons version (#2131) upgrade beanutils to 1.9.4 upgrade cli to 1.4 --- pom.xml | 8 +++++++- seatunnel-dist/release-docs/LICENSE | 4 +--- seatunnel-dist/release-docs/NOTICE | 24 +++++++++++++++++++++++ tools/dependencies/known-dependencies.txt | 4 +--- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 9339d5f40fb..a5d6b442ec7 100644 --- a/pom.xml +++ b/pom.xml @@ -112,7 +112,8 @@ 1.5.6 2.3.9 1.2 - 1.9.3 + 1.9.4 + 1.4 1.7 1.8.1 4.3.0 @@ -794,6 +795,11 @@ commons-beanutils ${commons.beanutils.version} + + commons-cli + commons-cli + ${commons.cli.version} + commons-configuration commons-configuration diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE index 2a9a3c9afa7..c8c137573e4 100644 --- a/seatunnel-dist/release-docs/LICENSE +++ b/seatunnel-dist/release-docs/LICENSE @@ -338,9 +338,7 @@ The text of each license is the standard Apache 2.0 license. (Apache License, Version 2.0) An open source Java toolkit for Amazon S3 (net.java.dev.jets3t:jets3t:0.9.0 - http://www.jets3t.org) (Apache License, Version 2.0) Apache Avro (org.apache.avro:avro:1.10.0 - https://avro.apache.org) (Apache License, Version 2.0) Apache Avro (org.apache.avro:avro:1.10.1 - https://avro.apache.org) - (Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.7.0 - https://commons.apache.org/proper/commons-beanutils/) - (Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.3 - https://commons.apache.org/proper/commons-beanutils/) - (Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.3.1 - http://commons.apache.org/proper/commons-cli/) + (Apache License, Version 2.0) Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/) (Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.4 - http://commons.apache.org/proper/commons-cli/) (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.13 - https://commons.apache.org/proper/commons-codec/) (Apache License, Version 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/collections/) diff --git a/seatunnel-dist/release-docs/NOTICE b/seatunnel-dist/release-docs/NOTICE index 564463f1ac3..9022476e003 100644 --- a/seatunnel-dist/release-docs/NOTICE +++ b/seatunnel-dist/release-docs/NOTICE @@ -4529,4 +4529,28 @@ The files - velocity-engine-scripting/src/main/java/org/apache/velocity/script/VelocityScriptEngineFactory.java are Copyright 2006 Sun Microsystems, Inc., and licenced under a BSD-like licence. +========================================================================= + +Apache Commons CLI NOTICE + +========================================================================= + +Apache Commons CLI +Copyright 2001-2017 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +========================================================================= + +Apache Commons BeanUtils NOTICE + +========================================================================= + +Apache Commons BeanUtils +Copyright 2000-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + ========================================================================= \ No newline at end of file diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 22d342d4cb9..3573a829282 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -77,9 +77,7 @@ clickhouse-http-client-0.3.2-patch9-shaded.jar clickhouse-http-client-0.3.2-patch9.jar clickhouse-jdbc-0.2.jar clickhouse-jdbc-0.3.2-patch9.jar -commons-beanutils-1.9.3.jar -commons-cli-1.2.jar -commons-cli-1.3.1.jar +commons-beanutils-1.9.4.jar commons-cli-1.4.jar commons-codec-1.13.jar commons-collections-3.2.2.jar From e7cfa2bc737cd15081807d0f6388f447ff6fbcaf Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 5 Jul 2022 18:32:11 +0800 Subject: [PATCH 13/76] [Connector-V2][Doc] Add File Sink Connector V2 document (#2120) * add file sink connector v2 * Update docs/en/connector-v2/sink/File.mdx Co-authored-by: Hisoka * Update docs/en/connector-v2/sink/File.mdx Co-authored-by: Hisoka --- docs/en/connector-v2/sink/File.mdx | 269 +++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 docs/en/connector-v2/sink/File.mdx diff --git a/docs/en/connector-v2/sink/File.mdx b/docs/en/connector-v2/sink/File.mdx new file mode 100644 index 00000000000..71e7631c6a2 --- /dev/null +++ b/docs/en/connector-v2/sink/File.mdx @@ -0,0 +1,269 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# File + +## Description + +Output data to local or hdfs or s3 file. + +:::tip + +Used to write data to file. Supports Batch and Streaming mode. + +* [x] Batch +* [x] Streaming + +::: + +## Options + + + + +| name | type | required | default value | +| --------------------------------- | ------ | -------- | ------------------------------------------------------------- | +| path | string | yes | - | +| file_name_expression | string | no | "${transactionId}" | +| file_format | string | no | "text" | +| filename_time_format | string | no | "yyyy.MM.dd" | +| field_delimiter | string | no | '\001' | +| row_delimiter | string | no | "\n" | +| partition_by | array | no | - | +| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | +| is_partition_field_write_in_file | boolean| no | false | +| sink_columns | array | no | When this parameter is empty, all fields are sink columns | +| is_enable_transaction | boolean| no | true | +| save_mode | string | no | "error" | + +### path [string] + +The target dir path is required. The `hdfs file` starts with `hdfs://` , and the `local file` starts with `file://`, + +### file_name_expression [string] + +`file_name_expression` describes the file expression which will be created into the `path`. We can add the variable `${now}` or `${uuid}` in the `file_name_expression`, like `test_${uuid}_${now}`, +`${now}` represents the current time, and its format can be defined by specifying the option `filename_time_format`. + +Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file. + +### file_format [string] + +We supported `file_format` is `text`. + +Please note that, The final file name will ends with the file_format's suffix, the suffix of the text file is `txt`. + +### filename_time_format [string] + +When the format in the `file_name_expression` parameter is `xxxx-${now}` , `filename_time_format` can specify the time format of the path, and the default value is `yyyy.MM.dd` . The commonly used time formats are listed as follows: + +| Symbol | Description | +| ------ | ------------------ | +| y | Year | +| M | Month | +| d | Day of month | +| H | Hour in day (0-23) | +| m | Minute in hour | +| s | Second in minute | + +See [Java SimpleDateFormat](https://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html) for detailed time format syntax. + +### field_delimiter [string] + +The separator between columns in a row of data. + +### row_delimiter [string] + +The separator between rows in a file. + +### partition_by [array] + +Partition data based on selected fields + +### partition_dir_expression [string] + +If the `partition_by` is specified, we will generate the corresponding partition directory based on the partition information, and the final file will be placed in the partition directory. + +Default `partition_dir_expression` is `${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/`. `k0` is the first partition field and `v0` is the value of the first partition field. + +### is_partition_field_write_in_file [boolean] + +If `is_partition_field_write_in_file` is `true`, the partition field and the value of it will be write into data file. + +For example, if you want to write a Hive Data File, Its value should be `false`. + +### sink_columns [array] + +Which columns need be write to file, default value is all of the columns get from `Transform` or `Source`. +The order of the fields determines the order in which the file is actually written. + +### is_enable_transaction [boolean] + +If `is_enable_transaction` is true, we will ensure that data will not be lost or duplicated when it is written to the target directory. + +Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file. + +Only support `true` now. + +### save_mode [string] + +Storage mode, currently supports `overwrite` , `append` , `ignore` and `error` . For the specific meaning of each mode, see [save-modes](https://spark.apache.org/docs/latest/sql-programming-guide.html#save-modes) + +Streaming Job not support `overwrite`. + + + + +| name | type | required | default value | +| --------------------------------- | ------ | -------- | ------------------------------------------------------------- | +| path | string | yes | - | +| file_name_expression | string | no | "${transactionId}" | +| file_format | string | no | "text" | +| filename_time_format | string | no | "yyyy.MM.dd" | +| field_delimiter | string | no | '\001' | +| row_delimiter | string | no | "\n" | +| partition_by | array | no | - | +| partition_dir_expression | string | no | "${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/" | +| is_partition_field_write_in_file | boolean| no | false | +| sink_columns | array | no | When this parameter is empty, all fields are sink columns | +| is_enable_transaction | boolean| no | true | +| save_mode | string | no | "error" | + +### path [string] + +The target dir path is required. The `hdfs file` starts with `hdfs://` , and the `local file` starts with `file://`, + +### file_name_expression [string] + +`file_name_expression` describes the file expression which will be created into the `path`. We can add the variable `${now}` or `${uuid}` in the `file_name_expression`, like `test_${uuid}_${now}`, +`${now}` represents the current time, and its format can be defined by specifying the option `filename_time_format`. + +Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file. + +### file_format [string] + +We supported `file_format` is `text`. + +Please note that, The final file name will ends with the file_format's suffix, the suffix of the text file is `txt`. + +### filename_time_format [string] + +When the format in the `file_name_expression` parameter is `xxxx-${now}` , `filename_time_format` can specify the time format of the path, and the default value is `yyyy.MM.dd` . The commonly used time formats are listed as follows: + +| Symbol | Description | +| ------ | ------------------ | +| y | Year | +| M | Month | +| d | Day of month | +| H | Hour in day (0-23) | +| m | Minute in hour | +| s | Second in minute | + +See [Java SimpleDateFormat](https://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html) for detailed time format syntax. + +### field_delimiter [string] + +The separator between columns in a row of data. + +### row_delimiter [string] + +The separator between rows in a file. + +### partition_by [array] + +Partition data based on selected fields + +### partition_dir_expression [string] + +If the `partition_by` is specified, we will generate the corresponding partition directory based on the partition information, and the final file will be placed in the partition directory. + +Default `partition_dir_expression` is `${k0}=${v0}/${k1}=${v1}/.../${kn}=${vn}/`. `k0` is the first partition field and `v0` is the value of the first partition field. + +### is_partition_field_write_in_file [boolean] + +If `is_partition_field_write_in_file` is `true`, the partition field and the value of it will be write into data file. + +For example, if you want to write a Hive Data File, Its value should be `false`. + +### sink_columns [array] + +Which columns need be write to file, default value is all of the columns get from `Transform` or `Source`. +The order of the fields determines the order in which the file is actually written. + +### is_enable_transaction [boolean] + +If `is_enable_transaction` is true, we will ensure that data will not be lost or duplicated when it is written to the target directory. + +Please note that, If `is_enable_transaction` is `true`, we will auto add `${transactionId}_` in the head of the file. + +Only support `true` now. + +### save_mode [string] + +Storage mode, currently supports `overwrite` , `append` , `ignore` and `error` . For the specific meaning of each mode, see [save-modes](https://spark.apache.org/docs/latest/sql-programming-guide.html#save-modes) + +Streaming Job not support `overwrite`. + + + +## Example + + + + +```bash + +LocalFile { + path="file:///tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="text" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" +} + +``` + + + + + +```bash + +HdfsFile { + path="file:///tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="text" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" +} + +``` + + + From e1e44266b4f4faff46b8eaa856277f421b177ab5 Mon Sep 17 00:00:00 2001 From: dijie Date: Tue, 5 Jul 2022 19:50:53 +0800 Subject: [PATCH 14/76] [seatunnel-1947][seatunnel-server] init & add interface for script/user (#2112) * [seatunnel-1947][seatunnel-server] init & add interface for script/script_param/user * [seatunnel-1947][seatunnel-server] update mysql connection * [seatunnel-1947][seatunnel-server] adjust for reviews. Co-authored-by: dijie --- pom.xml | 56 +++++ seatunnel-dist/release-docs/LICENSE | 179 +++++++++++++++- .../licenses/LICENSE-HikariCP.txt | 191 +++++++++++++++++ .../release-docs/licenses/LICENSE-mybatis.txt | 13 ++ .../release-docs/licenses/LICENSE-spring.txt | 125 +++++++++++ seatunnel-server/pom.xml | 32 +++ seatunnel-server/seatunnel-app/pom.xml | 170 +++++++++++++++ .../seatunnel/app/SeatunnelApplication.java | 38 ++++ .../app/common/DatasourceStatusEnum.java | 41 ++++ .../apache/seatunnel/app/common/Result.java | 97 +++++++++ .../app/common/ScriptParamStatusEnum.java | 40 ++++ .../app/common/ScriptStatusEnum.java | 41 ++++ .../seatunnel/app/common/ScriptTypeEnum.java | 40 ++++ .../app/common/SeatunnelErrorEnum.java | 51 +++++ .../app/common/SeatunnelException.java | 48 +++++ .../seatunnel/app/common/UserStatusEnum.java | 39 ++++ .../seatunnel/app/common/UserTypeEnum.java | 39 ++++ .../apache/seatunnel/app/config/Swagger2.java | 87 ++++++++ .../app/controller/ScriptController.java | 106 ++++++++++ .../app/controller/UserController.java | 99 +++++++++ .../seatunnel/app/dal/dao/IScriptDao.java | 40 ++++ .../app/dal/dao/IScriptParamDao.java | 31 +++ .../seatunnel/app/dal/dao/IUserDao.java | 40 ++++ .../app/dal/dao/impl/ScriptDaoImpl.java | 86 ++++++++ .../app/dal/dao/impl/ScriptParamDaoImpl.java | 64 ++++++ .../app/dal/dao/impl/UserDaoImpl.java | 95 +++++++++ .../seatunnel/app/dal/entity/Script.java | 45 ++++ .../seatunnel/app/dal/entity/ScriptParam.java | 39 ++++ .../apache/seatunnel/app/dal/entity/User.java | 39 ++++ .../app/dal/mapper/ScriptMapper.java | 38 ++++ .../app/dal/mapper/ScriptParamMapper.java | 34 +++ .../seatunnel/app/dal/mapper/UserMapper.java | 40 ++++ .../domain/dto/script/AddEmptyScriptDto.java | 31 +++ .../dto/script/CheckScriptDuplicateDto.java | 28 +++ .../app/domain/dto/script/ListScriptsDto.java | 28 +++ .../dto/script/UpdateScriptContentDto.java | 30 +++ .../dto/script/UpdateScriptParamDto.java | 30 +++ .../app/domain/dto/user/ListUserDto.java | 27 +++ .../app/domain/dto/user/UpdateUserDto.java | 31 +++ .../request/script/AddEmptyScriptReq.java | 37 ++++ .../domain/request/script/ScriptListReq.java | 35 ++++ .../script/UpdateScriptContentReq.java | 39 ++++ .../request/script/UpdateScriptParamReq.java | 34 +++ .../app/domain/request/user/AddUserReq.java | 24 +++ .../app/domain/request/user/BaseUserReq.java | 37 ++++ .../domain/request/user/UpdateUserReq.java | 32 +++ .../app/domain/request/user/UserListReq.java | 33 +++ .../response/script/AddEmptyScriptRes.java | 25 +++ .../response/script/BaseScriptInfoRes.java | 43 ++++ .../response/script/ScriptParamRes.java | 33 +++ .../response/script/ScriptSimpleInfoRes.java | 26 +++ .../app/domain/response/user/AddUserRes.java | 25 +++ .../domain/response/user/BaseUserInfoRes.java | 39 ++++ .../response/user/UserSimpleInfoRes.java | 24 +++ .../seatunnel/app/service/IScriptService.java | 44 ++++ .../seatunnel/app/service/IUserService.java | 41 ++++ .../app/service/impl/ScriptServiceImpl.java | 194 ++++++++++++++++++ .../app/service/impl/UserServiceImpl.java | 119 +++++++++++ .../app/util/GlobalExceptionHandler.java | 62 ++++++ .../apache/seatunnel/app/util/Md5Utils.java | 31 +++ .../seatunnel/app/util/PasswordUtils.java | 30 +++ .../src/main/resources/application.yml | 25 +++ .../src/main/resources/logback-spring.xml | 56 +++++ .../seatunnel/app/dal/mapper/ScriptMapper.xml | 89 ++++++++ .../app/dal/mapper/ScriptParamMapper.xml | 62 ++++++ .../seatunnel/app/dal/mapper/UserMapper.xml | 83 ++++++++ tools/dependencies/known-dependencies.txt | 61 ++++++ 67 files changed, 3708 insertions(+), 3 deletions(-) create mode 100644 seatunnel-dist/release-docs/licenses/LICENSE-HikariCP.txt create mode 100644 seatunnel-dist/release-docs/licenses/LICENSE-mybatis.txt create mode 100644 seatunnel-dist/release-docs/licenses/LICENSE-spring.txt create mode 100644 seatunnel-server/pom.xml create mode 100644 seatunnel-server/seatunnel-app/pom.xml create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/DatasourceStatusEnum.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/Result.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptParamStatusEnum.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptStatusEnum.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptTypeEnum.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeatunnelErrorEnum.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeatunnelException.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserStatusEnum.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserTypeEnum.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/Swagger2.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ScriptController.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IScriptDao.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IScriptParamDao.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IUserDao.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/ScriptDaoImpl.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/ScriptParamDaoImpl.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/impl/UserDaoImpl.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/Script.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/ScriptParam.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/entity/User.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/ScriptMapper.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/ScriptParamMapper.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/mapper/UserMapper.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/dto/script/AddEmptyScriptDto.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/dto/script/CheckScriptDuplicateDto.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/dto/script/ListScriptsDto.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/dto/script/UpdateScriptContentDto.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/dto/script/UpdateScriptParamDto.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/dto/user/ListUserDto.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/dto/user/UpdateUserDto.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/script/AddEmptyScriptReq.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/script/ScriptListReq.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/script/UpdateScriptContentReq.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/script/UpdateScriptParamReq.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/user/AddUserReq.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/user/BaseUserReq.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/user/UpdateUserReq.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/request/user/UserListReq.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/script/AddEmptyScriptRes.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/script/BaseScriptInfoRes.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/script/ScriptParamRes.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/script/ScriptSimpleInfoRes.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/user/AddUserRes.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/user/BaseUserInfoRes.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/domain/response/user/UserSimpleInfoRes.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IScriptService.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/IUserService.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/ScriptServiceImpl.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/UserServiceImpl.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/util/GlobalExceptionHandler.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/util/Md5Utils.java create mode 100644 seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/util/PasswordUtils.java create mode 100644 seatunnel-server/seatunnel-app/src/main/resources/application.yml create mode 100644 seatunnel-server/seatunnel-app/src/main/resources/logback-spring.xml create mode 100644 seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptMapper.xml create mode 100644 seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/ScriptParamMapper.xml create mode 100644 seatunnel-server/seatunnel-app/src/main/resources/org/apache/seatunnel/app/dal/mapper/UserMapper.xml diff --git a/pom.xml b/pom.xml index a5d6b442ec7..2378419b73b 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ seatunnel-plugin-discovery seatunnel-formats seatunnel-dist + seatunnel-server @@ -198,6 +199,12 @@ org.apache.seatunnel.shade 4.3.0 1.1.8.3 + 2.6.8 + 2.2.2 + 1.2.9 + 2.6.1 + 1.5.10 + 6.2.2.Final @@ -846,6 +853,55 @@ ${javax.servlet.jap.version} + + + org.springframework.boot + spring-boot-starter-web + ${spring-boot.version} + + + + org.springframework.boot + spring-boot-starter-jetty + ${spring-boot.version} + + + + org.mybatis.spring.boot + mybatis-spring-boot-starter + ${mybatis-spring-boot-starter.version} + + + + com.alibaba + druid-spring-boot-starter + ${druid-spring-boot-starter.version} + + + + io.springfox + springfox-swagger2 + ${springfox-swagger.version} + + + + io.springfox + springfox-swagger-ui + ${springfox-swagger.version} + + + + io.swagger + swagger-annotations + ${swagger-annotations.version} + + + + org.hibernate.validator + hibernate-validator + ${hibernate.validator.version} + + diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE index c8c137573e4..3a24746fb1c 100644 --- a/seatunnel-dist/release-docs/LICENSE +++ b/seatunnel-dist/release-docs/LICENSE @@ -275,6 +275,8 @@ The text of each license is the standard Apache 2.0 license. (Apache 2) org.roaringbitmap:shims (org.roaringbitmap:shims:0.9.22 - https://github.com/RoaringBitmap/RoaringBitmap) (Apache 2) scalaj-http (org.scalaj:scalaj-http_2.11:2.3.0 - http://github.com/scalaj/scalaj-http) (Apache 2) univocity-parsers (com.univocity:univocity-parsers:2.7.3 - http://github.com/univocity/univocity-parsers) + (Apache 2) druid (com.alibaba:druid:1.2.9 - https://github.com/alibaba/druid) + (Apache 2) druid-spring-boot-starter (com.alibaba:druid-spring-boot-starter:1.2.9 - https://github.com/alibaba/druid) (Apache 2.0 License) Apache Mina SSHD :: Common support utilities (org.apache.sshd:sshd-common:2.7.0 - https://www.apache.org/sshd/sshd-common/) (Apache 2.0 License) Apache Mina SSHD :: Core (org.apache.sshd:sshd-core:2.7.0 - https://www.apache.org/sshd/sshd-core/) (Apache 2.0 License) Apache Mina SSHD :: SCP (org.apache.sshd:sshd-scp:2.7.0 - https://www.apache.org/sshd/sshd-scp/) @@ -323,7 +325,12 @@ The text of each license is the standard Apache 2.0 license. (Apache License 2.0) jDBI (org.jdbi:jdbi:2.63.1 - http://jdbi.org/) (Apache License 2.0) snappy (org.iq80.snappy:snappy:0.2 - http://github.com/dain/snappy) (Apache License 2.0) snappy (org.iq80.snappy:snappy:0.3 - http://github.com/dain/snappy) - (Apache License Version 2) Jetty SSLEngine (org.mortbay.jetty:jetty-sslengine:6.1.26 - http://jetty.mortbay.org) + (Apache License 2.0) Hibernate Validator Engine (org.hibernate.validator:hibernate-validator:6.2.2.Final - http://hibernate.org/validator/hibernate-validator) + (Apache License 2.0) Jakarta Bean Validation API (jakarta.validation:jakarta.validation-api:2.0.2 - https://beanvalidation.org) + (Apache License 2.0) swagger-annotations (io.swagger:swagger-annotations:1.5.10 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations) + (Apache License 2.0) swagger-models (io.swagger:swagger-models:1.5.10 - https://github.com/swagger-api/swagger-core/modules/swagger-models) + (Apache License 2.0) swagger-models (io.swagger:swagger-models:1.5.10 - https://github.com/swagger-api/swagger-core/modules/swagger-models) + (Apache License Version 2) HikariCP (com.zaxxer:HikariCP:4.0.3 - https://github.com/brettwooldridge/HikariCP) (Apache License) HttpClient (commons-httpclient:commons-httpclient:3.1 - http://jakarta.apache.org/httpcomponents/httpclient-3.x/) (Apache License) HttpClient (org.apache.httpcomponents:httpclient:4.0.1 - http://hc.apache.org/httpcomponents-client) (Apache License) HttpClient (org.apache.httpcomponents:httpclient:4.2.5 - http://hc.apache.org/httpcomponents-client) @@ -578,6 +585,29 @@ The text of each license is the standard Apache 2.0 license. (Apache License, Version 2.0) jmockdata (com.github.jsonzou:jmockdata:4.3.0 - https://github.com/jsonzou/jmockdata) (Apache License, Version 2.0) stream-lib (com.clearspring.analytics:stream:2.7.0 - https://github.com/addthis/stream-lib) (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.2.1.Final - http://www.jboss.org) + (Apache License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:31.0.1-jre - https://github.com/google/guava) + (Apache License, Version 2.0) SnakeYAML (org.yaml:snakeyaml:1.29 - http://www.snakeyaml.org) + (Apache License, Version 2.0) Spring AOP (org.springframework:spring-aop:5.3.20 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) Spring Beans (org.springframework:spring-beans:5.3.20 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) Spring Commons Logging Bridge (org.springframework:spring-jcl:5.3.20 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) Spring Context (org.springframework:spring-context:5.3.20 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) Spring Core (org.springframework:spring-core:5.3.20 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) Spring JDBC (org.springframework:spring-jdbc:5.3.15 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) Spring Plugin - Core (org.springframework.plugin:spring-plugin-core:1.2.0.RELEASE - https://github.com/spring-projects/spring-plugin/spring-plugin-core) + (Apache License, Version 2.0) Spring Plugin - Metadata Extension (org.springframework.plugin:spring-plugin-metadata:1.2.0.RELEASE - https://github.com/spring-projects/spring-plugin/spring-plugin-metadata) + (Apache License, Version 2.0) Spring Transaction (org.springframework:spring-tx:5.3.15 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) Spring Web (org.springframework:spring-web:5.3.20 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) Spring Web MVC (org.springframework:spring-webmvc:5.3.20 - https://github.com/spring-projects/spring-framework) + (Apache License, Version 2.0) spring-boot (org.springframework.boot:spring-boot:2.6.8 - https://spring.io/projects/spring-boot) + (Apache License, Version 2.0) spring-boot-autoconfigure (org.springframework.boot:spring-boot-autoconfigure:2.6.8 - https://spring.io/projects/spring-boot) + (Apache License, Version 2.0) spring-boot-starter (org.springframework.boot:spring-boot-starter:2.6.8 - https://spring.io/projects/spring-boot) + (Apache License, Version 2.0) spring-boot-starter-jdbc (org.springframework.boot:spring-boot-starter-jdbc:2.6.3 - https://spring.io/projects/spring-boot) + (Apache License, Version 2.0) spring-boot-starter-jetty (org.springframework.boot:spring-boot-starter-jetty:2.6.8 - https://spring.io/projects/spring-boot) + (Apache License, Version 2.0) spring-boot-starter-json (org.springframework.boot:spring-boot-starter-json:2.6.8 - https://spring.io/projects/spring-boot) + (Apache License, Version 2.0) spring-boot-starter-logging (org.springframework.boot:spring-boot-starter-logging:2.6.8 - https://spring.io/projects/spring-boot) + (Apache License, Version 2.0) spring-boot-starter-web (org.springframework.boot:spring-boot-starter-web:2.6.8 - https://spring.io/projects/spring-boot) + (Apache License, Version 2.0) tomcat-embed-el (org.apache.tomcat.embed:tomcat-embed-el:9.0.63 - https://tomcat.apache.org/) + (Apache License, version 2.0) JBoss Logging 3 (org.jboss.logging:jboss-logging:3.4.1.Final - http://www.jboss.org) (Apache v2) BoneCP :: Core Library (com.jolbox:bonecp:0.8.0.RELEASE - http://jolbox.com/bonecp) (Apache-2.0) json4s-ast (org.json4s:json4s-ast_2.11:3.5.3 - https://github.com/json4s/json4s) (Apache-2.0) json4s-core (org.json4s:json4s-core_2.11:3.5.3 - https://github.com/json4s/json4s) @@ -605,6 +635,7 @@ The text of each license is the standard Apache 2.0 license. (The Apache License, Version 2.0) org.jetbrains.kotlin:kotlin-stdlib-jdk7 (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.4.10 - https://kotlinlang.org/) (The Apache License, Version 2.0) org.jetbrains.kotlin:kotlin-stdlib-jdk8 (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.4.10 - https://kotlinlang.org/) (The Apache License, Version 2.0) software.amazon.ion:ion-java (software.amazon.ion:ion-java:1.0.2 - https://github.com/amznlabs/ion-java/) + (The Apache License, Version 2.0) com.github.jsonzou:jmockdata (com.github.jsonzou:jmockdata:4.3.0 - https://github.com/jsonzou/jmockdata) (The Apache Software License, Version 2.0) ASM based accessors helper used by json-smart (net.minidev:accessors-smart:1.2 - http://www.minidev.net/) (The Apache Software License, Version 2.0) Aggregate Designer Algorithm (net.hydromatic:aggdesigner-algorithm:6.0 - http://github.com/julianhyde/aggdesigner/aggdesigner-algorithm) (The Apache Software License, Version 2.0) Apache Avro (org.apache.avro:avro:1.7.7 - http://avro.apache.org) @@ -924,7 +955,64 @@ The text of each license is the standard Apache 2.0 license. (The Apache Software License, Version 2.0) jasper-runtime (tomcat:jasper-runtime:5.5.23 - http://tomcat.apache.org/jasper-runtime) (The Apache Software License, Version 2.0) JAX-RS provider for JSON content type (org.codehaus.jackson:jackson-jaxrs:1.9.13 - http://jackson.codehaus.org) (The Apache Software License, Version 2.0) Xml Compatibility extensions for Jackson (org.codehaus.jackson:jackson-xc:1.9.13 - http://jackson.codehaus.org) - + + (The Apache Software License, Version 2.0) ClassMate (com.fasterxml:classmate:1.3.1 - http://github.com/cowtowncoder/java-classmate) + (The Apache Software License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:11.0.2 - http://code.google.com/p/guava-libraries/guava) + (The Apache Software License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:13.0.1 - http://code.google.com/p/guava-libraries/guava) + (The Apache Software License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:16.0.1 - http://code.google.com/p/guava-libraries/guava) + (The Apache Software License, Version 2.0) Guava: Google Core Libraries for Java (com.google.guava:guava:18.0 - http://code.google.com/p/guava-libraries/guava) + (The Apache Software License, Version 2.0) HikariCP (com.zaxxer:HikariCP:4.0.3 - https://github.com/brettwooldridge/HikariCP) + (The Apache Software License, Version 2.0) Jackson datatype: JSR310 (com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.3 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jsr310) + (The Apache Software License, Version 2.0) Jackson datatype: jdk8 (com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.3 - https://github.com/FasterXML/jackson-modules-java8/jackson-datatype-jdk8) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.10.5 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.11.0 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.11.4 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.4.5 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.6.7 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.7.0 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.7.8 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-annotations (com.fasterxml.jackson.core:jackson-annotations:2.9.2 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.10.5 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.11.0 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.11.4 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.13.3 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.6.7 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.7.3 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.7.9 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.8.10 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.8.11 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-core (com.fasterxml.jackson.core:jackson-core:2.9.2 - https://github.com/FasterXML/jackson-core) + (The Apache Software License, Version 2.0) Jackson-module-parameter-names (com.fasterxml.jackson.module:jackson-module-parameter-names:2.13.3 - https://github.com/FasterXML/jackson-modules-java8/jackson-module-parameter-names) + (The Apache Software License, Version 2.0) MapStruct Core (org.mapstruct:mapstruct:1.0.0.Final - http://mapstruct.org/mapstruct/) + (The Apache Software License, Version 2.0) Snappy for Java (org.xerial.snappy:snappy-java:1.0.5 - http://github.com/xerial/snappy-java/) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.12.6 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-module-scala (com.fasterxml.jackson.module:jackson-module-scala_2.11:2.12.6 - http://wiki.fasterxml.com/JacksonModuleScala) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.10.5.1 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.11.0 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.11.4 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.12.6 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.13.3 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.6.7.1 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.7.3 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-databind (com.fasterxml.jackson.core:jackson-databind:2.9.2 - http://github.com/FasterXML/jackson) + (The Apache Software License, Version 2.0) jackson-module-scala (com.fasterxml.jackson.module:jackson-module-scala_2.11:2.6.7.1 - http://wiki.fasterxml.com/JacksonModuleScala) + (The Apache Software License, Version 2.0) mybatis (org.mybatis:mybatis:3.5.9 - http://www.mybatis.org/mybatis-3) + (The Apache Software License, Version 2.0) mybatis-spring (org.mybatis:mybatis-spring:2.0.7 - http://www.mybatis.org/spring/) + (The Apache Software License, Version 2.0) mybatis-spring-boot-autoconfigure (org.mybatis.spring.boot:mybatis-spring-boot-autoconfigure:2.2.2 - http://www.mybatis.org/spring-boot-starter/mybatis-spring-boot-autoconfigure/) + (The Apache Software License, Version 2.0) mybatis-spring-boot-starter (org.mybatis.spring.boot:mybatis-spring-boot-starter:2.2.2 - http://www.mybatis.org/spring-boot-starter/mybatis-spring-boot-starter/) + (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.2.6 - https://github.com/xerial/snappy-java) + (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.4 - https://github.com/xerial/snappy-java) + (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.1 - https://github.com/xerial/snappy-java) + (The Apache Software License, Version 2.0) snappy-java (org.xerial.snappy:snappy-java:1.1.7.3 - https://github.com/xerial/snappy-java) + (The Apache Software License, Version 2.0) springfox-core (io.springfox:springfox-core:2.6.1 - https://github.com/springfox/springfox) + (The Apache Software License, Version 2.0) springfox-schema (io.springfox:springfox-schema:2.6.1 - https://github.com/springfox/springfox) + (The Apache Software License, Version 2.0) springfox-spi (io.springfox:springfox-spi:2.6.1 - https://github.com/springfox/springfox) + (The Apache Software License, Version 2.0) springfox-spring-web (io.springfox:springfox-spring-web:2.6.1 - https://github.com/springfox/springfox) + (The Apache Software License, Version 2.0) springfox-swagger-common (io.springfox:springfox-swagger-common:2.6.1 - https://github.com/springfox/springfox) + (The Apache Software License, Version 2.0) springfox-swagger-ui (io.springfox:springfox-swagger-ui:2.6.1 - https://github.com/springfox/springfox) + (The Apache Software License, Version 2.0) springfox-swagger2 (io.springfox:springfox-swagger2:2.6.1 - https://github.com/springfox/springfox) + + ======================================================================== MIT License ======================================================================== @@ -945,6 +1033,16 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.25 - http://www.slf4j.org) (MIT License) pyrolite (net.razorvine:pyrolite:4.13 - https://github.com/irmen/Pyrolite) (MIT License) scopt (com.github.scopt:scopt_2.11:3.5.0 - https://github.com/scopt/scopt) + (MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:1.7.36 - http://www.slf4j.org) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.6.4 - http://www.slf4j.org) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.15 - http://www.slf4j.org) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.16 - http://www.slf4j.org) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.21 - http://www.slf4j.org) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.28 - http://www.slf4j.org) + (MIT License) SLF4J API Module (org.slf4j:slf4j-api:1.7.30 - http://www.slf4j.org) + (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.10 - http://www.slf4j.org) + (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.15 - http://www.slf4j.org) + (MIT License) SLF4J LOG4J-12 Binding (org.slf4j:slf4j-log4j12:1.7.16 - http://www.slf4j.org) (MIT) Jedis (redis.clients:jedis:3.2.0 - https://github.com/xetorthio/jedis) (MIT-License) spoiwo (com.norbitltd:spoiwo_2.11:1.8.0 - https://github.com/norbert-radyk/spoiwo/) (The MIT License (MIT)) influxdb java bindings (org.influxdb:influxdb-java:2.22 - http://www.influxdb.org) @@ -1014,6 +1112,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (New BSD License) janino (org.codehaus.janino:janino:3.0.8 - http://janino-compiler.github.io/janino/) (New BSD License) janino (org.codehaus.janino:janino:3.0.9 - http://janino-compiler.github.io/janino/) (New BSD license) Protocol Buffer Java API (com.google.protobuf:protobuf-java:2.5.0 - http://code.google.com/p/protobuf) + (New BSD License) Hamcrest Core (org.hamcrest:hamcrest-core:1.3 - https://github.com/hamcrest/JavaHamcrest/hamcrest-core) (Revised BSD) JSch (com.jcraft:jsch:0.1.54 - http://www.jcraft.com/jsch/) (The BSD 3-Clause License) leveldbjni-all (org.fusesource.leveldbjni:leveldbjni-all:1.8 - http://leveldbjni.fusesource.org/leveldbjni-all) (The BSD License) ANTLR 4 Runtime (org.antlr:antlr4-runtime:4.5.1 - http://www.antlr.org/antlr4-runtime) @@ -1026,7 +1125,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (The BSD License) xmlenc Library (xmlenc:xmlenc:0.52 - http://xmlenc.sourceforge.net) (The BSD License) ASM Core (asm:asm:3.1 - http://asm.objectweb.org/asm/) (The BSD License) ASM Commons (asm:asm-commons:3.1 - http://asm.objectweb.org/asm-commons/) - (The BSD License) ASM Tree (asm:asm-tree:3.1 - http://asm.objectweb.org/asm-tree/) + (The BSD License) ASM Tree (asm:asm-tree:3.1 - http://asm.objectweb.org/asm-tree/) (New BSD License) Commons Compiler (org.codehaus.janino:commons-compiler:2.7.6 - http://docs.codehaus.org/display/JANINO/Home/commons-compiler) (New BSD License) Janino (org.codehaus.janino:janino:3.1.6 - http://docs.codehaus.org/display/JANINO/Home/janino) (BSD licence) ANTLR ST4 4.0.4 (org.antlr:ST4:4.0.4 - http://www.stringtemplate.org) @@ -1092,6 +1191,80 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0) (GNU General Public Library) Streaming API for XML (javax.xml.stream:stax-api:1.0-2 - no url defined) + (CDDL License) Expression Language 3.0 (org.glassfish:javax.el:3.0.0 - http://el-spec.java.net) + (CDDL License) Expression Language 3.0 (org.glassfish:javax.el:3.0.1-b12 - http://uel.java.net) + (CDDL License) Expression Language 3.0 API (javax.el:javax.el-api:3.0.0 - http://uel-spec.java.net) + (CDDL License) HK2 API module (org.glassfish.hk2:hk2-api:2.4.0-b34 - https://hk2.java.net/hk2-api) + (CDDL License) HK2 API module (org.glassfish.hk2:hk2-api:2.5.0-b32 - https://hk2.java.net/hk2-api) + (CDDL License) HK2 Implementation Utilities (org.glassfish.hk2:hk2-utils:2.4.0-b34 - https://hk2.java.net/hk2-utils) + (CDDL License) HK2 Implementation Utilities (org.glassfish.hk2:hk2-utils:2.5.0-b32 - https://hk2.java.net/hk2-utils) + (CDDL License) JSP implementation (org.glassfish.web:javax.servlet.jsp:2.3.2 - http://jsp.java.net) + (CDDL License) Java Servlet API (javax.servlet.jsp:jsp-api:2.1 - https://javaee.github.io/javaee-jsp-api) + (CDDL License) Java Servlet API (javax.servlet:javax.servlet-api:3.1.0 - http://servlet-spec.java.net) + (CDDL License) Java Servlet API (javax.servlet:servlet-api:2.5 - http://servlet-spec.java.net) + (CDDL License) Java Transaction API (javax.transaction:jta:1.1 - http://java.sun.com/products/jta) + (CDDL License) OSGi resource locator bundle - used by various API providers that rely on META-INF/services mechanism to locate providers. (org.glassfish.hk2:osgi-resource-locator:1.0.1 - http://glassfish.org/osgi-resource-locator/) + (CDDL License) ServiceLocator Default Implementation (org.glassfish.hk2:hk2-locator:2.4.0-b34 - https://hk2.java.net/hk2-locator) + (CDDL License) ServiceLocator Default Implementation (org.glassfish.hk2:hk2-locator:2.5.0-b32 - https://hk2.java.net/hk2-locator) + (CDDL License) aopalliance version 1.0 repackaged as a module (org.glassfish.hk2.external:aopalliance-repackaged:2.4.0-b34 - https://hk2.java.net/external/aopalliance-repackaged) + (CDDL License) aopalliance version 1.0 repackaged as a module (org.glassfish.hk2.external:aopalliance-repackaged:2.5.0-b32 - https://hk2.java.net/external/aopalliance-repackaged) + (CDDL License) javax.annotation API (javax.annotation:javax.annotation-api:1.2 - http://jcp.org/en/jsr/detail?id=250) + (CDDL License) javax.annotation API (javax.annotation:javax.annotation-api:1.3.2 - http://jcp.org/en/jsr/detail?id=250) + (CDDL License) javax.inject:1 as OSGi bundle (org.glassfish.hk2.external:javax.inject:2.4.0-b34 - https://hk2.java.net/external/javax.inject) + (CDDL License) javax.inject:1 as OSGi bundle (org.glassfish.hk2.external:javax.inject:2.5.0-b32 - https://hk2.java.net/external/javax.inject) + (CDDL License) jersey-container-servlet (org.glassfish.jersey.containers:jersey-container-servlet:2.22.2 - https://jersey.java.net/project/jersey-container-servlet/) + (CDDL License) jersey-container-servlet-core (org.glassfish.jersey.containers:jersey-container-servlet-core:2.22.2 - https://jersey.java.net/project/jersey-container-servlet-core/) + (CDDL License) jersey-container-servlet-core (org.glassfish.jersey.containers:jersey-container-servlet-core:2.25.1 - https://jersey.java.net/project/jersey-container-servlet-core/) + (CDDL License) jersey-core-client (org.glassfish.jersey.core:jersey-client:2.22.2 - https://jersey.java.net/jersey-client/) + (CDDL License) jersey-core-client (org.glassfish.jersey.core:jersey-client:2.25.1 - https://jersey.java.net/jersey-client/) + (CDDL License) jersey-core-common (org.glassfish.jersey.core:jersey-common:2.22.2 - https://jersey.java.net/jersey-common/) + (CDDL License) jersey-core-common (org.glassfish.jersey.core:jersey-common:2.25.1 - https://jersey.java.net/jersey-common/) + (CDDL License) jersey-core-server (org.glassfish.jersey.core:jersey-server:2.22.2 - https://jersey.java.net/jersey-server/) + (CDDL License) jersey-core-server (org.glassfish.jersey.core:jersey-server:2.25.1 - https://jersey.java.net/jersey-server/) + (CDDL License) jersey-media-jaxb (org.glassfish.jersey.media:jersey-media-jaxb:2.22.2 - https://jersey.java.net/project/jersey-media-jaxb/) + (CDDL License) jersey-media-jaxb (org.glassfish.jersey.media:jersey-media-jaxb:2.25.1 - https://jersey.java.net/project/jersey-media-jaxb/) + (CDDL License) jersey-repackaged-guava (org.glassfish.jersey.bundles.repackaged:jersey-guava:2.22.2 - https://jersey.java.net/project/project/jersey-guava/) + (CDDL License) jersey-repackaged-guava (org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1 - https://jersey.java.net/project/project/jersey-guava/) + (CDDL License) JavaBeans Activation Framework (com.sun.activation:javax.activation:1.2.0 - http://java.net/all/javax.activation/) + (CDDL License) JavaBeans Activation Framework API jar (javax.activation:javax.activation-api:1.2.0 - http://java.net/all/javax.activation-api/) + (CDDL License) JavaMail API (com.sun.mail:javax.mail:1.5.6 - http://javamail.java.net/javax.mail) + (CDDL+GPL License) jersey-container-servlet (org.glassfish.jersey.containers:jersey-container-servlet:2.22.2 - https://jersey.java.net/project/jersey-container-servlet/) + (CDDL+GPL License) jersey-container-servlet-core (org.glassfish.jersey.containers:jersey-container-servlet-core:2.22.2 - https://jersey.java.net/project/jersey-container-servlet-core/) + (CDDL+GPL License) jersey-container-servlet-core (org.glassfish.jersey.containers:jersey-container-servlet-core:2.25.1 - https://jersey.java.net/project/jersey-container-servlet-core/) + (CDDL+GPL License) jersey-core-client (org.glassfish.jersey.core:jersey-client:2.22.2 - https://jersey.java.net/jersey-client/) + (CDDL+GPL License) jersey-core-client (org.glassfish.jersey.core:jersey-client:2.25.1 - https://jersey.java.net/jersey-client/) + (CDDL+GPL License) jersey-core-common (org.glassfish.jersey.core:jersey-common:2.22.2 - https://jersey.java.net/jersey-common/) + (CDDL+GPL License) jersey-core-common (org.glassfish.jersey.core:jersey-common:2.25.1 - https://jersey.java.net/jersey-common/) + (CDDL+GPL License) jersey-core-server (org.glassfish.jersey.core:jersey-server:2.22.2 - https://jersey.java.net/jersey-server/) + (CDDL+GPL License) jersey-core-server (org.glassfish.jersey.core:jersey-server:2.25.1 - https://jersey.java.net/jersey-server/) + (CDDL+GPL License) jersey-media-jaxb (org.glassfish.jersey.media:jersey-media-jaxb:2.22.2 - https://jersey.java.net/project/jersey-media-jaxb/) + (CDDL+GPL License) jersey-media-jaxb (org.glassfish.jersey.media:jersey-media-jaxb:2.25.1 - https://jersey.java.net/project/jersey-media-jaxb/) + (CDDL+GPL License) jersey-repackaged-guava (org.glassfish.jersey.bundles.repackaged:jersey-guava:2.22.2 - https://jersey.java.net/project/project/jersey-guava/) + (CDDL+GPL License) jersey-repackaged-guava (org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1 - https://jersey.java.net/project/project/jersey-guava/) + (CDDL + GPLv2 with classpath exception) Expression Language 3.0 (org.glassfish:javax.el:3.0.0 - http://el-spec.java.net) + (CDDL + GPLv2 with classpath exception) Expression Language 3.0 (org.glassfish:javax.el:3.0.1-b12 - http://uel.java.net) + (CDDL + GPLv2 with classpath exception) Expression Language 3.0 API (javax.el:javax.el-api:3.0.0 - http://uel-spec.java.net) + (CDDL + GPLv2 with classpath exception) HK2 API module (org.glassfish.hk2:hk2-api:2.4.0-b34 - https://hk2.java.net/hk2-api) + (CDDL + GPLv2 with classpath exception) HK2 API module (org.glassfish.hk2:hk2-api:2.5.0-b32 - https://hk2.java.net/hk2-api) + (CDDL + GPLv2 with classpath exception) HK2 Implementation Utilities (org.glassfish.hk2:hk2-utils:2.4.0-b34 - https://hk2.java.net/hk2-utils) + (CDDL + GPLv2 with classpath exception) HK2 Implementation Utilities (org.glassfish.hk2:hk2-utils:2.5.0-b32 - https://hk2.java.net/hk2-utils) + (CDDL + GPLv2 with classpath exception) JSP implementation (org.glassfish.web:javax.servlet.jsp:2.3.2 - http://jsp.java.net) + (CDDL + GPLv2 with classpath exception) Java Servlet API (javax.servlet.jsp:jsp-api:2.1 - https://javaee.github.io/javaee-jsp-api) + (CDDL + GPLv2 with classpath exception) Java Servlet API (javax.servlet:javax.servlet-api:3.1.0 - http://servlet-spec.java.net) + (CDDL + GPLv2 with classpath exception) Java Servlet API (javax.servlet:servlet-api:2.5 - http://servlet-spec.java.net) + (CDDL + GPLv2 with classpath exception) Java Transaction API (javax.transaction:jta:1.1 - http://java.sun.com/products/jta) + (CDDL + GPLv2 with classpath exception) OSGi resource locator bundle - used by various API providers that rely on META-INF/services mechanism to locate providers. (org.glassfish.hk2:osgi-resource-locator:1.0.1 - http://glassfish.org/osgi-resource-locator/) + (CDDL + GPLv2 with classpath exception) ServiceLocator Default Implementation (org.glassfish.hk2:hk2-locator:2.4.0-b34 - https://hk2.java.net/hk2-locator) + (CDDL + GPLv2 with classpath exception) ServiceLocator Default Implementation (org.glassfish.hk2:hk2-locator:2.5.0-b32 - https://hk2.java.net/hk2-locator) + (CDDL + GPLv2 with classpath exception) aopalliance version 1.0 repackaged as a module (org.glassfish.hk2.external:aopalliance-repackaged:2.4.0-b34 - https://hk2.java.net/external/aopalliance-repackaged) + (CDDL + GPLv2 with classpath exception) aopalliance version 1.0 repackaged as a module (org.glassfish.hk2.external:aopalliance-repackaged:2.5.0-b32 - https://hk2.java.net/external/aopalliance-repackaged) + (CDDL + GPLv2 with classpath exception) javax.annotation API (javax.annotation:javax.annotation-api:1.2 - http://jcp.org/en/jsr/detail?id=250) + (CDDL + GPLv2 with classpath exception) javax.annotation API (javax.annotation:javax.annotation-api:1.3.2 - http://jcp.org/en/jsr/detail?id=250) + (CDDL + GPLv2 with classpath exception) javax.inject:1 as OSGi bundle (org.glassfish.hk2.external:javax.inject:2.4.0-b34 - https://hk2.java.net/external/javax.inject) + (CDDL + GPLv2 with classpath exception) javax.inject:1 as OSGi bundle (org.glassfish.hk2.external:javax.inject:2.5.0-b32 - https://hk2.java.net/external/javax.inject) + (CDDL/GPLv2+CE) JavaBeans Activation Framework (com.sun.activation:javax.activation:1.2.0 - http://java.net/all/javax.activation/) + (CDDL/GPLv2+CE) JavaBeans Activation Framework API jar (javax.activation:javax.activation-api:1.2.0 - http://java.net/all/javax.activation-api/) + (CDDL/GPLv2+CE) JavaMail API (com.sun.mail:javax.mail:1.5.6 - http://javamail.java.net/javax.mail) ======================================================================== Eclipse Public License diff --git a/seatunnel-dist/release-docs/licenses/LICENSE-HikariCP.txt b/seatunnel-dist/release-docs/licenses/LICENSE-HikariCP.txt new file mode 100644 index 00000000000..8405e89a0b1 --- /dev/null +++ b/seatunnel-dist/release-docs/licenses/LICENSE-HikariCP.txt @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/seatunnel-dist/release-docs/licenses/LICENSE-mybatis.txt b/seatunnel-dist/release-docs/licenses/LICENSE-mybatis.txt new file mode 100644 index 00000000000..9c311afbf06 --- /dev/null +++ b/seatunnel-dist/release-docs/licenses/LICENSE-mybatis.txt @@ -0,0 +1,13 @@ +Copyright ${license.git.copyrightYears} the original author or authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/seatunnel-dist/release-docs/licenses/LICENSE-spring.txt b/seatunnel-dist/release-docs/licenses/LICENSE-spring.txt new file mode 100644 index 00000000000..512235b40b6 --- /dev/null +++ b/seatunnel-dist/release-docs/licenses/LICENSE-spring.txt @@ -0,0 +1,125 @@ +# Spring RTS game engine + +## LICENSE + +Spring is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 2 of the License, or +(at your option) any later version. + +Spring is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +For details about the licenses, +see [GNU GPL v2 (gpl-2.0.txt)](gpl-2.0.html) +and [v3 (gpl-3.0.txt)](gpl-3.0.html). + +For a list of authors of this software, see [AUTHORS](AUTHORS.html). + +## Third Parties + +### file SOP.hpp +Copyright (c) 2012, Daniel Cornel. Published on drivenbynostalgia.com. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +### gflags +Copyright (c) 2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +### mingw-std-threads +Copyright (c) 2016, Mega Limited +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +### moodycamel's ConcurrentQueue +Copyright (c) 2013-2016, Cameron Desrochers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this list of +conditions and the following disclaimer in the documentation and/or other materials +provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/seatunnel-server/pom.xml b/seatunnel-server/pom.xml new file mode 100644 index 00000000000..dc936151072 --- /dev/null +++ b/seatunnel-server/pom.xml @@ -0,0 +1,32 @@ + + + + + seatunnel + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-server + pom + + seatunnel-app + + + \ No newline at end of file diff --git a/seatunnel-server/seatunnel-app/pom.xml b/seatunnel-server/seatunnel-app/pom.xml new file mode 100644 index 00000000000..eeb6b7ced5a --- /dev/null +++ b/seatunnel-server/seatunnel-app/pom.xml @@ -0,0 +1,170 @@ + + + + + seatunnel-server + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-app + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-tomcat + + + log4j-to-slf4j + org.apache.logging.log4j + + + + + + org.springframework.boot + spring-boot-starter-jetty + + + org.eclipse.jetty.websocket + javax-websocket-server-impl + + + org.eclipse.jetty.websocket + websocket-server + + + + + + org.mybatis.spring.boot + mybatis-spring-boot-starter + ${mybatis-spring-boot-starter.version} + + + spring-beans + org.springframework + + + spring-boot-autoconfigure + org.springframework.boot + + + spring-core + org.springframework + + + spring-boot-starter + org.springframework.boot + + + + + + io.springfox + springfox-swagger2 + ${springfox-swagger.version} + + + spring-aop + org.springframework + + + spring-beans + org.springframework + + + spring-context + org.springframework + + + + + + io.springfox + springfox-swagger-ui + ${springfox-swagger.version} + + + + io.swagger + swagger-annotations + ${swagger-annotations.version} + + + + org.hibernate.validator + hibernate-validator + + + classmate + com.fasterxml + + + + + + org.apache.commons + commons-lang3 + + + + mysql + mysql-connector-java + provided + + + + org.scala-lang + scala-library + provided + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + + + repackage + + + + + + + \ No newline at end of file diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java new file mode 100644 index 00000000000..757e863fdba --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/SeatunnelApplication.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app; + +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +@SpringBootApplication +@EnableTransactionManagement +@EnableConfigurationProperties +@EnableScheduling +@EnableAsync(proxyTargetClass = true) +@MapperScan({"org.apache.seatunnel.app.dal"}) +public class SeatunnelApplication { + public static void main(String[] args) { + SpringApplication.run(SeatunnelApplication.class, args); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/DatasourceStatusEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/DatasourceStatusEnum.java new file mode 100644 index 00000000000..b271a5b0433 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/DatasourceStatusEnum.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +public enum DatasourceStatusEnum { + ONLINE(1, "online"), + OFFLINE(2, "offline"), + DELETED(3, "deleted"), + ; + + private final int code; + private final String description; + + DatasourceStatusEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/Result.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/Result.java new file mode 100644 index 00000000000..79d7ae12b85 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/Result.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +public class Result { + + private static final Result OK = success(); + + private int code = 0; + + private String msg; + + private T data; + + private Result() { + this.data = null; + } + + private Result(SeatunnelErrorEnum errorEnum) { + this.code = errorEnum.getCode(); + this.msg = errorEnum.getMsg(); + this.data = null; + } + + private Result(SeatunnelErrorEnum errorEnum, String... messages) { + this.code = errorEnum.getCode(); + this.msg = String.format(errorEnum.getTemplate(), messages); + this.data = null; + } + + public static Result success() { + return new Result<>(); + } + + public static Result success(T data) { + Result result = success(); + result.setData(data); + return result; + } + + public static Result failure(SeatunnelErrorEnum errorEnum) { + Result result = new Result<>(errorEnum); + return result; + } + + public static Result failure(SeatunnelErrorEnum errorEnum, String... messages) { + Result result = new Result<>(errorEnum, messages); + return result; + } + + public boolean isSuccess() { + return OK.getCode() == this.code; + } + + public boolean isFailed() { + return !this.isSuccess(); + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public T getData() { + return data; + } + + public void setData(T data) { + this.data = data; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptParamStatusEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptParamStatusEnum.java new file mode 100644 index 00000000000..56195dd8de8 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptParamStatusEnum.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +public enum ScriptParamStatusEnum { + NORMAL(0, "normal"), + DELETED(1, "deleted"), + ; + + private final int code; + private final String description; + + ScriptParamStatusEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptStatusEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptStatusEnum.java new file mode 100644 index 00000000000..17704a217b0 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptStatusEnum.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +public enum ScriptStatusEnum { + UNPUBLISHED(0, "unpublished"), + PUBLISHED(1, "published"), + DELETED(2, "deleted"), + ; + + private final int code; + private final String description; + + ScriptStatusEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptTypeEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptTypeEnum.java new file mode 100644 index 00000000000..89d627c6041 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/ScriptTypeEnum.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +public enum ScriptTypeEnum { + OFFLINE(0, "offline"), + REALTIME(1, "realtime"), + ; + + private final int code; + private final String description; + + ScriptTypeEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeatunnelErrorEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeatunnelErrorEnum.java new file mode 100644 index 00000000000..3f770b7256b --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeatunnelErrorEnum.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +public enum SeatunnelErrorEnum { + + SCRIPT_ALREADY_EXIST(10001, "script already exist", "You already have a script with the same name : '%s'"), + NO_SUCH_SCRIPT(10002, "no such script", "No such script. Maybe deleted by others."), + USER_ALREADY_EXISTS(10003, "user already exist", "The same username [%s] is exist."), + NO_SUCH_USER(10002, "no such user", "No such user. Maybe deleted by others."), + ILLEGAL_STATE(99998, "illegal state", "%s"), + UNKNOWN(99999, "unknown exception", "unknown exception") + ; + + private final int code; + private final String msg; + private final String template; + + SeatunnelErrorEnum(int code, String msg, String template) { + this.code = code; + this.msg = msg; + this.template = template; + } + + public int getCode() { + return code; + } + + public String getMsg() { + return msg; + } + + public String getTemplate() { + return template; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeatunnelException.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeatunnelException.java new file mode 100644 index 00000000000..e683296311c --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/SeatunnelException.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +import java.util.Arrays; + +public class SeatunnelException extends RuntimeException{ + private SeatunnelErrorEnum errorEnum; + + public SeatunnelException(SeatunnelErrorEnum e) { + super(e.getMsg()); + this.errorEnum = e; + } + + public SeatunnelException(SeatunnelErrorEnum e, String... msg) { + super(e.getMsg().concat(" ").concat(Arrays.toString(msg))); + this.errorEnum = e; + } + + public static SeatunnelException newInstance(SeatunnelErrorEnum e, String... msg) { + return new SeatunnelException(e, msg); + + } + + public static SeatunnelException newInstance(SeatunnelErrorEnum e) { + return new SeatunnelException(e); + + } + + public SeatunnelErrorEnum getErrorEnum() { + return errorEnum; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserStatusEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserStatusEnum.java new file mode 100644 index 00000000000..c62b7e1cf73 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserStatusEnum.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +public enum UserStatusEnum { + ENABLE(0, "enable"), + DISABLE(1, "disable"), + ; + private final int code; + private final String description; + + UserStatusEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserTypeEnum.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserTypeEnum.java new file mode 100644 index 00000000000..1b050c51697 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/common/UserTypeEnum.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.common; + +public enum UserTypeEnum { + NORMAL(0, "normal"), + ; + + private final int code; + private final String description; + + UserTypeEnum(int code, String description) { + this.code = code; + this.description = description; + } + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/Swagger2.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/Swagger2.java new file mode 100644 index 00000000000..f8a5718da5a --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/config/Swagger2.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.config; + +import com.fasterxml.classmate.TypeResolver; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import springfox.documentation.builders.ApiInfoBuilder; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.service.ApiInfo; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +@Configuration +@EnableSwagger2 +@EnableWebMvc +public class Swagger2 implements WebMvcConfigurer { + @Autowired + private TypeResolver typeResolver; + + @Bean + public Docket createRestApi() { + + return new Docket(DocumentationType.SWAGGER_2) + .apiInfo(apiInfo()) + .select() + .apis(RequestHandlerSelectors.basePackage("org.apache.seatunnel.app.controller")) + .paths(PathSelectors.any()) + .build(); + + } + + @Override + public void addViewControllers(ViewControllerRegistry registry) { + registry.addRedirectViewController("/api/v2/api-docs", "/v2/api-docs"); + registry.addRedirectViewController("/api/swagger-resources/configuration/ui", "/swagger-resources/configuration/ui"); + registry.addRedirectViewController("/api/swagger-resources/configuration/security", "/swagger-resources/configuration/security"); + registry.addRedirectViewController("/api/swagger-resources", "/swagger-resources"); + + registry.addRedirectViewController("/api/null/api-docs", + "/api-docs").setKeepQueryParams(true); + registry.addRedirectViewController("/api/null/swagger-resources/configuration/ui", + "/swagger-resources/configuration/ui"); + registry.addRedirectViewController("/api/null/swagger-resources/configuration/security", + "/swagger-resources/configuration/security"); + registry.addRedirectViewController("/api/null/swagger-resources", "/swagger-resources"); + } + + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + registry.addResourceHandler("/api/swagger-ui.html**").addResourceLocations("classpath:/META-INF/resources/swagger-ui.html"); + registry.addResourceHandler("/api/webjars/**").addResourceLocations("classpath:/META-INF/resources/webjars/"); + registry.addResourceHandler("/doc.html**").addResourceLocations("classpath:/META-INF/resources/"); + } + + private ApiInfo apiInfo() { + return new ApiInfoBuilder() + .title("seatunnel api docs") + .version("1.0.0") + .description("API description of Seatunnel") + .termsOfServiceUrl("https://seatunnel.apache.org/") + .build(); + } + +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ScriptController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ScriptController.java new file mode 100644 index 00000000000..a17d0fc12b9 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/ScriptController.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.controller; + +import org.apache.seatunnel.app.common.Result; +import org.apache.seatunnel.app.domain.request.script.AddEmptyScriptReq; +import org.apache.seatunnel.app.domain.request.script.ScriptListReq; +import org.apache.seatunnel.app.domain.request.script.UpdateScriptContentReq; +import org.apache.seatunnel.app.domain.request.script.UpdateScriptParamReq; +import org.apache.seatunnel.app.domain.response.script.AddEmptyScriptRes; +import org.apache.seatunnel.app.domain.response.script.ScriptParamRes; +import org.apache.seatunnel.app.domain.response.script.ScriptSimpleInfoRes; +import org.apache.seatunnel.app.service.IScriptService; + +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; +import javax.validation.constraints.NotNull; + +import java.util.List; + +@RequestMapping("/api/v1/script") +@RestController +public class ScriptController { + @Resource + private IScriptService iScriptService; + + @PostMapping("/script") + @ApiOperation(value = "add an empty script", httpMethod = "POST") + public Result addEmptyScript(@RequestBody @NotNull AddEmptyScriptReq addEmptyScriptReq) { + return Result.success(iScriptService.addEmptyScript(addEmptyScriptReq)); + } + + @PutMapping("/scriptContent") + @ApiOperation(value = "update script", httpMethod = "PUT") + public Result updateScriptContent(@RequestBody @NotNull UpdateScriptContentReq updateScriptContentReq) { + iScriptService.updateScriptContent(updateScriptContentReq); + return Result.success(); + } + + @DeleteMapping("/script") + @ApiOperation(value = "delete script", httpMethod = "DELETE") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "script id", dataType = "Integer"), + }) + public Result delete(@RequestParam @NotNull Integer id) { + iScriptService.delete(id); + return Result.success(); + } + + @PostMapping("/list") + @ApiOperation(value = "script list", httpMethod = "POST") + public Result> list(@RequestBody @NotNull ScriptListReq scriptListReq) { + return Result.success(iScriptService.list(scriptListReq)); + } + + @GetMapping("/scriptContent") + @ApiOperation(value = "fetch script content", httpMethod = "GET") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "script id", dataType = "Integer"), + }) + public Result fetchScriptContent(@RequestParam @NotNull Integer id) { + return Result.success(iScriptService.fetchScriptContent(id)); + } + + @PutMapping("/scriptParam") + @ApiOperation(value = "update script param", httpMethod = "PUT") + public Result updateScriptParam(@RequestBody @NotNull UpdateScriptParamReq updateScriptParamReq) { + iScriptService.updateScriptParam(updateScriptParamReq); + return Result.success(); + } + + @GetMapping("/scriptParam") + @ApiOperation(value = "fetch script param", httpMethod = "GET") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "script id", dataType = "Integer"), + }) + public Result> fetchScriptParam(@RequestParam @NotNull Integer id) { + return Result.success(iScriptService.fetchScriptParam(id)); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java new file mode 100644 index 00000000000..ec5c16d714a --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.controller; + +import org.apache.seatunnel.app.common.Result; +import org.apache.seatunnel.app.domain.request.user.AddUserReq; +import org.apache.seatunnel.app.domain.request.user.UpdateUserReq; +import org.apache.seatunnel.app.domain.request.user.UserListReq; +import org.apache.seatunnel.app.domain.response.user.AddUserRes; +import org.apache.seatunnel.app.domain.response.user.UserSimpleInfoRes; +import org.apache.seatunnel.app.service.IUserService; + +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; +import javax.validation.constraints.NotNull; + +import java.util.List; + +@RequestMapping("/api/v1/user") +@RestController +public class UserController { + + @Resource + private IUserService iUserService; + + @PostMapping("/user") + @ApiOperation(value = "add user", httpMethod = "POST") + public Result add(@RequestBody @NotNull AddUserReq addReq) { + return Result.success(iUserService.add(addReq)); + } + + @PutMapping("/user") + @ApiOperation(value = "update user", httpMethod = "PUT") + public Result update(@RequestBody @NotNull UpdateUserReq updateReq) { + iUserService.update(updateReq); + return Result.success(); + } + + @DeleteMapping("/user") + @ApiOperation(value = "delete user", httpMethod = "DELETE") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "user id", dataType = "Integer"), + }) + public Result delete(@RequestParam @NotNull Integer id) { + iUserService.delete(id); + return Result.success(); + } + + @PostMapping("/list") + @ApiOperation(value = "user list", httpMethod = "POST") + public Result> list(@RequestBody @NotNull UserListReq userListReq) { + return Result.success(iUserService.list(userListReq)); + } + + @PutMapping("/enable") + @ApiOperation(value = "enable a user", httpMethod = "PUT") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "user id", dataType = "Integer"), + }) + public Result enable(@RequestParam @NotNull Integer id) { + iUserService.enable(id); + return Result.success(); + } + + @PutMapping("/disable") + @ApiOperation(value = "disable a user", httpMethod = "PUT") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "user id", dataType = "Integer"), + }) + public Result disable(@RequestParam @NotNull Integer id) { + iUserService.disable(id); + return Result.success(); + } +} diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IScriptDao.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IScriptDao.java new file mode 100644 index 00000000000..7da0332ac69 --- /dev/null +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/dal/dao/IScriptDao.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.app.dal.dao; + +import org.apache.seatunnel.app.dal.entity.Script; +import org.apache.seatunnel.app.domain.dto.script.AddEmptyScriptDto; +import org.apache.seatunnel.app.domain.dto.script.CheckScriptDuplicateDto; +import org.apache.seatunnel.app.domain.dto.script.ListScriptsDto; +import org.apache.seatunnel.app.domain.dto.script.UpdateScriptContentDto; + +import java.util.List; + +public interface IScriptDao { + void checkScriptDuplicate(CheckScriptDuplicateDto dto); + + int addEmptyScript(AddEmptyScriptDto dto); + + Script getScript(Integer id); + + void updateScriptContent(UpdateScriptContentDto dto); + + void deleteScript(int id); + + List