From b41ad33544350c6dbfb03be7265f49ced219c6b7 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Tue, 6 Sep 2022 17:26:05 +0800 Subject: [PATCH 01/22] [Engine][Example] Add seatunnel example module --- .../core/starter/config/ConfigChecker.java | 4 +- .../flink/config/FlinkApiConfigChecker.java | 2 +- .../core/starter/seatunnel/SeaTunnel.java | 33 ++++++++++++ .../seatunnel/args/SeaTunnelCommandArgs.java | 16 +++++- .../SeaTunnelApiConfValidateCommand.java | 52 +++++++++++++++++++ .../SeaTunnelApiTaskExecuteCommand.java} | 29 +++++++---- .../command/SeaTunnelCommandBuilder.java | 33 ++++++++++++ .../config/SeaTunnelApiConfigChecker.java | 31 +++++++++++ .../spark/command/SparkCommandBuilder.java | 13 +---- .../spark/config/SparkApiConfigChecker.java | 2 +- .../seatunnel-engine-examples/pom.xml | 7 +++ .../engine/SeaTunnelEngineExample.java | 52 +++++++++++++++++++ .../resources/examples/fake_to_console.conf | 51 ++++++++++++++++++ 13 files changed, 299 insertions(+), 26 deletions(-) create mode 100644 seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java create mode 100644 seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java rename seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/{SeaTunnelStarter.java => command/SeaTunnelApiTaskExecuteCommand.java} (71%) create mode 100644 seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java create mode 100644 seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java create mode 100644 seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java create mode 100644 seatunnel-examples/seatunnel-engine-examples/src/main/resources/examples/fake_to_console.conf diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/ConfigChecker.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/ConfigChecker.java index 96925569c22..3512eb38ab3 100644 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/ConfigChecker.java +++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/ConfigChecker.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.core.starter.config; -import org.apache.seatunnel.apis.base.env.RuntimeEnv; import org.apache.seatunnel.core.starter.exception.ConfigCheckException; import org.apache.seatunnel.shade.com.typesafe.config.Config; @@ -25,9 +24,8 @@ /** * Check the config is valid. * - * @param the environment type. */ -public interface ConfigChecker { +public interface ConfigChecker { /** * Check if the config is validated, if check fails, throw exception. diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiConfigChecker.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiConfigChecker.java index 2ae68515d0a..4a41e8b11d2 100644 --- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiConfigChecker.java +++ b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiConfigChecker.java @@ -22,7 +22,7 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; -public class FlinkApiConfigChecker implements ConfigChecker { +public class FlinkApiConfigChecker implements ConfigChecker { @Override public void checkConfig(Config config) throws ConfigCheckException { diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java new file mode 100644 index 00000000000..a3a98a7b184 --- /dev/null +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel; + +import org.apache.seatunnel.core.starter.Seatunnel; +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.exception.CommandException; +import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.command.SeaTunnelCommandBuilder; + +public class SeaTunnel { + public static void main(String[] args) throws CommandException { + SeaTunnelCommandArgs seaTunnelCommandArgs = CommandLineUtils.parseSeaTunnelArgs(args); + Command command = + new SeaTunnelCommandBuilder().buildCommand(seaTunnelCommandArgs); + Seatunnel.run(command); + } +} diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java index 19c5d8587c3..45e1962685f 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java @@ -21,15 +21,29 @@ import org.apache.seatunnel.core.starter.command.AbstractCommandArgs; import org.apache.seatunnel.core.starter.config.EngineType; +import com.beust.jcommander.Parameter; + import java.util.List; public class SeaTunnelCommandArgs extends AbstractCommandArgs { /** - * Undefined parameters parsed will be stored here as seatunnel engint command parameters. + * Undefined parameters parsed will be stored here as seatunnel engine command parameters. */ private List seatunnelParams; + @Parameter(names = {"-n", "--name"}, + description = "The name of job") + private String name = "seatunnel_job"; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + @Override public EngineType getEngineType() { return EngineType.SEATUNNEL; diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java new file mode 100644 index 00000000000..8036120ae44 --- /dev/null +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.command; + +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.config.ConfigBuilder; +import org.apache.seatunnel.core.starter.exception.ConfigCheckException; +import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.config.SeaTunnelApiConfigChecker; +import org.apache.seatunnel.core.starter.utils.FileUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; + +/** + * Use to validate the configuration of the SeaTunnel API. + */ +public class SeaTunnelApiConfValidateCommand implements Command { + + private static final Logger LOGGER = LoggerFactory.getLogger(SeaTunnelApiConfValidateCommand.class); + + private final SeaTunnelCommandArgs seaTunnelCommandArgs; + + public SeaTunnelApiConfValidateCommand(SeaTunnelCommandArgs seaTunnelCommandArgs) { + this.seaTunnelCommandArgs = seaTunnelCommandArgs; + } + + @Override + public void execute() throws ConfigCheckException { + Path configPath = FileUtils.getConfigPath(seaTunnelCommandArgs); + ConfigBuilder configBuilder = new ConfigBuilder(configPath); + new SeaTunnelApiConfigChecker().checkConfig(configBuilder.getConfig()); + LOGGER.info("config OK !"); + } +} diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelStarter.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java similarity index 71% rename from seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelStarter.java rename to seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java index 94335cca925..ac9cfe13d0a 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelStarter.java +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.seatunnel.core.starter.seatunnel; +package org.apache.seatunnel.core.starter.seatunnel.command; -import org.apache.seatunnel.common.config.Common; -import org.apache.seatunnel.common.config.DeployMode; +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.exception.CommandExecuteException; import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; import org.apache.seatunnel.core.starter.utils.FileUtils; import org.apache.seatunnel.engine.client.SeaTunnelClient; @@ -32,13 +32,23 @@ import java.nio.file.Path; import java.util.concurrent.ExecutionException; -public class SeaTunnelStarter { - public static void main(String[] args) { - SeaTunnelCommandArgs seaTunnelCommandArgs = CommandLineUtils.parseSeaTunnelArgs(args); +/** + * This command is used to execute the SeaTunnel engine job by SeaTunnel API. + */ +public class SeaTunnelApiTaskExecuteCommand implements Command { + + private final SeaTunnelCommandArgs seaTunnelCommandArgs; + + public SeaTunnelApiTaskExecuteCommand(SeaTunnelCommandArgs seaTunnelCommandArgs) { + this.seaTunnelCommandArgs = seaTunnelCommandArgs; + } + + @Override + public void execute() throws CommandExecuteException { Path configFile = FileUtils.getConfigPath(seaTunnelCommandArgs); - Common.setDeployMode(DeployMode.CLIENT); + JobConfig jobConfig = new JobConfig(); - jobConfig.setName("fake_to_file"); + jobConfig.setName(seaTunnelCommandArgs.getName()); ClientConfig clientConfig = ConfigProvider.locateAndGetClientConfig(); SeaTunnelClient engineClient = new SeaTunnelClient(clientConfig); @@ -49,7 +59,8 @@ public static void main(String[] args) { clientJobProxy = jobExecutionEnv.execute(); clientJobProxy.waitForJobComplete(); } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException(e); + throw new CommandExecuteException("SeaTunnel job executed failed", e); } } + } diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java new file mode 100644 index 00000000000..722c84e7b0d --- /dev/null +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.command; + +import org.apache.seatunnel.common.config.Common; +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.command.CommandBuilder; +import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; + +public class SeaTunnelCommandBuilder implements CommandBuilder { + + @Override + public Command buildCommand(SeaTunnelCommandArgs commandArgs) { + Common.setDeployMode(commandArgs.getDeployMode()); + return commandArgs.isCheckConfig() ? new SeaTunnelApiConfValidateCommand(commandArgs) + : new SeaTunnelApiTaskExecuteCommand(commandArgs); + } +} diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java new file mode 100644 index 00000000000..bfaff2a9c35 --- /dev/null +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.config; + +import org.apache.seatunnel.core.starter.config.ConfigChecker; +import org.apache.seatunnel.core.starter.exception.ConfigCheckException; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +public class SeaTunnelApiConfigChecker implements ConfigChecker { + + @Override + public void checkConfig(Config config) throws ConfigCheckException { + // TODO implement + } +} diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkCommandBuilder.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkCommandBuilder.java index 9f6a9ec5bab..9bf8e376e44 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkCommandBuilder.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkCommandBuilder.java @@ -27,18 +27,9 @@ public class SparkCommandBuilder implements CommandBuilder { @Override public Command buildCommand(SparkCommandArgs commandArgs) { Common.setDeployMode(commandArgs.getDeployMode()); - return new SeaTunnelApiCommandBuilder().buildCommand(commandArgs); + return commandArgs.isCheckConfig() ? new SparkApiConfValidateCommand(commandArgs) + : new SparkApiTaskExecuteCommand(commandArgs); } - /** - * Used to generate command for seaTunnel API. - */ - private static class SeaTunnelApiCommandBuilder extends SparkCommandBuilder { - @Override - public Command buildCommand(SparkCommandArgs commandArgs) { - return commandArgs.isCheckConfig() ? new SparkApiConfValidateCommand(commandArgs) - : new SparkApiTaskExecuteCommand(commandArgs); - } - } } diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java index b08a4a752c2..afdf775796a 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java @@ -22,7 +22,7 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; -public class SparkApiConfigChecker implements ConfigChecker { +public class SparkApiConfigChecker implements ConfigChecker { @Override public void checkConfig(Config config) throws ConfigCheckException { diff --git a/seatunnel-examples/seatunnel-engine-examples/pom.xml b/seatunnel-examples/seatunnel-engine-examples/pom.xml index 9c3a5ca5d13..e90526f149a 100644 --- a/seatunnel-examples/seatunnel-engine-examples/pom.xml +++ b/seatunnel-examples/seatunnel-engine-examples/pom.xml @@ -27,4 +27,11 @@ 4.0.0 seatunnel-engine-examples + + + org.apache.seatunnel + seatunnel-seatunnel-starter + ${revision} + + \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java new file mode 100644 index 00000000000..a50f312712b --- /dev/null +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.example.engine; + +import org.apache.seatunnel.core.starter.Seatunnel; +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.exception.CommandException; +import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.command.SeaTunnelCommandBuilder; + +import java.io.FileNotFoundException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; + +public class SeaTunnelEngineExample { + + public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { + String configFile = getTestConfigFile("/examples/fake_to_console.conf"); + SeaTunnelCommandArgs seaTunnelCommandArgs = new SeaTunnelCommandArgs(); + seaTunnelCommandArgs.setConfigFile(configFile); + seaTunnelCommandArgs.setCheckConfig(false); + seaTunnelCommandArgs.setName("fake_to_console"); + Command command = + new SeaTunnelCommandBuilder().buildCommand(seaTunnelCommandArgs); + Seatunnel.run(command); + } + + public static String getTestConfigFile(String configFile) throws FileNotFoundException, URISyntaxException { + URL resource = SeaTunnelEngineExample.class.getResource(configFile); + if (resource == null) { + throw new FileNotFoundException("Can't find config file: " + configFile); + } + return Paths.get(resource.toURI()).toString(); + } + +} diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/resources/examples/fake_to_console.conf b/seatunnel-examples/seatunnel-engine-examples/src/main/resources/examples/fake_to_console.conf new file mode 100644 index 00000000000..404c9ec6907 --- /dev/null +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/resources/examples/fake_to_console.conf @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + # You can set engine configuration here + execution.parallelism = 3 + job.mode = "BATCH" + execution.checkpoint.interval = 5000 + #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + FakeSource { + result_table_name = "fake" + field_name = "name,age", + parallelism = 2 + } + + FakeSource { + result_table_name = "fake" + field_name = "name,age", + parallelism = 3 + } +} + +transform { +} + +sink { + console { + source_table_name="fake" + } +} \ No newline at end of file From c174877978794733c1f871efbe4a51554f0c5889 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Tue, 6 Sep 2022 17:27:13 +0800 Subject: [PATCH 02/22] [Engine][Example] Add seatunnel e2e module --- seatunnel-e2e/seatunnel-engine-e2e/pom.xml | 7 ++++ .../pom.xml | 36 +++++++++++++++++++ .../seatunnel-engine-e2e-base/pom.xml | 36 +++++++++++++++++++ .../seatunnel/engine/e2e/TestUtils.java | 0 .../engine/e2e/engine/JobExecutionIT.java | 7 ++-- .../engine/e2e/engine/SeaTunnelContainer.java | 21 +++++++++++ .../resources/batch_fakesource_to_file.conf | 0 .../batch_fakesource_to_file_complex.conf | 0 .../streaming_fakesource_to_file_complex.conf | 0 9 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml rename seatunnel-e2e/seatunnel-engine-e2e/{ => seatunnel-engine-e2e-base}/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{ => seatunnel-engine-e2e-base}/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java (97%) create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/SeaTunnelContainer.java rename seatunnel-e2e/seatunnel-engine-e2e/{ => seatunnel-engine-e2e-base}/src/test/resources/batch_fakesource_to_file.conf (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{ => seatunnel-engine-e2e-base}/src/test/resources/batch_fakesource_to_file_complex.conf (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{ => seatunnel-engine-e2e-base}/src/test/resources/streaming_fakesource_to_file_complex.conf (100%) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/pom.xml index ea8a31f3acf..ea38a9b374f 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/pom.xml @@ -27,6 +27,12 @@ 4.0.0 seatunnel-engine-e2e + pom + + + seatunnel-engine-e2e-base + seatunnel-engine-connector-file-e2e + @@ -69,6 +75,7 @@ org.awaitility awaitility + test \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml new file mode 100644 index 00000000000..1b73ea3a07f --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml @@ -0,0 +1,36 @@ + + + + + + seatunnel-engine-e2e + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-engine-connector-v2-e2e + + + 8 + 8 + + + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml new file mode 100644 index 00000000000..8ea876392cc --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml @@ -0,0 +1,36 @@ + + + + + + seatunnel-engine-e2e + org.apache.seatunnel + ${revision} + + 4.0.0 + + seatunnel-engine-e2e-base + + + 8 + 8 + + + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java diff --git a/seatunnel-e2e/seatunnel-engine-e2e/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java similarity index 97% rename from seatunnel-e2e/seatunnel-engine-e2e/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java index 8fb21b51b7c..84a247bbfcb 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java @@ -17,8 +17,6 @@ package org.apache.seatunnel.engine.e2e.engine; -import static org.awaitility.Awaitility.await; - import org.apache.seatunnel.common.config.Common; import org.apache.seatunnel.common.config.DeployMode; import org.apache.seatunnel.engine.client.SeaTunnelClient; @@ -35,6 +33,7 @@ import com.google.common.collect.Lists; import com.hazelcast.client.config.ClientConfig; import com.hazelcast.instance.impl.HazelcastInstanceFactory; +import org.awaitility.Awaitility; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -88,7 +87,7 @@ public void testExecuteJob() { return clientJobProxy.waitForJobComplete(); }); - await().atMost(20000, TimeUnit.MILLISECONDS) + Awaitility.await().atMost(20000, TimeUnit.MILLISECONDS) .untilAsserted(() -> Assert.assertTrue( objectCompletableFuture.isDone() && JobStatus.FINISHED.equals(objectCompletableFuture.get()))); @@ -121,7 +120,7 @@ public void cancelJobTest() { Thread.sleep(1000); clientJobProxy.cancelJob(); - await().atMost(20000, TimeUnit.MILLISECONDS) + Awaitility.await().atMost(20000, TimeUnit.MILLISECONDS) .untilAsserted(() -> Assert.assertTrue( objectCompletableFuture.isDone() && JobStatus.CANCELED.equals(objectCompletableFuture.get()))); diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/SeaTunnelContainer.java new file mode 100644 index 00000000000..0e8d200aa7a --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/SeaTunnelContainer.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.e2e.engine; + +public abstract class SeaTunnelContainer { +} diff --git a/seatunnel-e2e/seatunnel-engine-e2e/src/test/resources/batch_fakesource_to_file.conf b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/batch_fakesource_to_file.conf similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/src/test/resources/batch_fakesource_to_file.conf rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/batch_fakesource_to_file.conf diff --git a/seatunnel-e2e/seatunnel-engine-e2e/src/test/resources/batch_fakesource_to_file_complex.conf b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/src/test/resources/batch_fakesource_to_file_complex.conf rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf diff --git a/seatunnel-e2e/seatunnel-engine-e2e/src/test/resources/streaming_fakesource_to_file_complex.conf b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/src/test/resources/streaming_fakesource_to_file_complex.conf rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf From 581f5763a6f685a68865991a6c359afaa08d048c Mon Sep 17 00:00:00 2001 From: Hisoka Date: Tue, 6 Sep 2022 19:16:14 +0800 Subject: [PATCH 03/22] [Engine] [Core] Add seatunnel engine example --- .../seatunnel-seatunnel-starter/pom.xml | 6 ++++ .../args/ExecutionModeConverter.java | 34 +++++++++++++++++++ .../seatunnel/args/SeaTunnelCommandArgs.java | 14 ++++++++ .../SeaTunnelApiTaskExecuteCommand.java | 26 ++++++++++++++ .../engine/client/job/ClientJobProxy.java | 21 +++++------- .../seatunnel-engine-examples/pom.xml | 13 ++++++- 6 files changed, 101 insertions(+), 13 deletions(-) create mode 100644 seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java diff --git a/seatunnel-core/seatunnel-seatunnel-starter/pom.xml b/seatunnel-core/seatunnel-seatunnel-starter/pom.xml index d2dd1ddbe63..b4d7c3bf607 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/pom.xml +++ b/seatunnel-core/seatunnel-seatunnel-starter/pom.xml @@ -46,6 +46,12 @@ seatunnel-engine-client ${project.version} + + + org.apache.seatunnel + seatunnel-engine-server + ${project.version} + \ No newline at end of file diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java new file mode 100644 index 00000000000..ee7da538b9f --- /dev/null +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.args; + +import org.apache.seatunnel.engine.common.runtime.ExecutionMode; + +import com.beust.jcommander.IStringConverter; +import com.beust.jcommander.ParameterException; + +public class ExecutionModeConverter implements IStringConverter { + @Override + public ExecutionMode convert(String value) { + try { + return ExecutionMode.valueOf(value.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new ParameterException("execution-mode: " + value + " is not allowed."); + } + } +} diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java index 45e1962685f..deeb470dcfc 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java @@ -20,6 +20,7 @@ import org.apache.seatunnel.common.config.DeployMode; import org.apache.seatunnel.core.starter.command.AbstractCommandArgs; import org.apache.seatunnel.core.starter.config.EngineType; +import org.apache.seatunnel.engine.common.runtime.ExecutionMode; import com.beust.jcommander.Parameter; @@ -36,6 +37,19 @@ public class SeaTunnelCommandArgs extends AbstractCommandArgs { description = "The name of job") private String name = "seatunnel_job"; + @Parameter(names = {"-e", "--deploy-mode"}, + description = "SeaTunnel deploy mode", + converter = ExecutionModeConverter.class) + private ExecutionMode executionMode = ExecutionMode.LOCAL; + + public ExecutionMode getExecutionMode() { + return executionMode; + } + + public void setExecutionMode(ExecutionMode executionMode) { + this.executionMode = executionMode; + } + public String getName() { return name; } diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java index ac9cfe13d0a..7d74706ef37 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java @@ -26,8 +26,13 @@ import org.apache.seatunnel.engine.client.job.JobExecutionEnvironment; import org.apache.seatunnel.engine.common.config.ConfigProvider; import org.apache.seatunnel.engine.common.config.JobConfig; +import org.apache.seatunnel.engine.common.config.SeaTunnelConfig; +import org.apache.seatunnel.engine.common.runtime.ExecutionMode; +import org.apache.seatunnel.engine.server.SeaTunnelNodeContext; import com.hazelcast.client.config.ClientConfig; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.instance.impl.HazelcastInstanceFactory; import java.nio.file.Path; import java.util.concurrent.ExecutionException; @@ -39,6 +44,9 @@ public class SeaTunnelApiTaskExecuteCommand implements Command { PassiveCompletableFuture jobFuture = doWaitForJobComplete(); @@ -84,18 +84,15 @@ public JobStatus waitForJobComplete() { jobImmutableInformation.getJobConfig().getName(), jobImmutableInformation.getJobId(), jobStatus)); + this.seaTunnelHazelcastClient.getHazelcastInstance().shutdown(); return jobStatus; } @Override public PassiveCompletableFuture doWaitForJobComplete() { - PassiveCompletableFuture jobFuture = - seaTunnelHazelcastClient.requestOnMasterAndGetCompletableFuture( - SeaTunnelWaitForJobCompleteCodec.encodeRequest(jobImmutableInformation.getJobId()), - response -> { - return JobStatus.values()[SeaTunnelWaitForJobCompleteCodec.decodeResponse(response)]; - }); - return jobFuture; + return seaTunnelHazelcastClient.requestOnMasterAndGetCompletableFuture( + SeaTunnelWaitForJobCompleteCodec.encodeRequest(jobImmutableInformation.getJobId()), + response -> JobStatus.values()[SeaTunnelWaitForJobCompleteCodec.decodeResponse(response)]); } @Override @@ -110,7 +107,7 @@ public void cancelJob() { public JobStatus getJobStatus() { int jobStatusOrdinal = seaTunnelHazelcastClient.requestOnMasterAndDecodeResponse( SeaTunnelGetJobStatusCodec.encodeRequest(jobImmutableInformation.getJobId()), - response -> SeaTunnelGetJobStatusCodec.decodeResponse(response)); + SeaTunnelGetJobStatusCodec::decodeResponse); return JobStatus.values()[jobStatusOrdinal]; } diff --git a/seatunnel-examples/seatunnel-engine-examples/pom.xml b/seatunnel-examples/seatunnel-engine-examples/pom.xml index e90526f149a..b75a68b856c 100644 --- a/seatunnel-examples/seatunnel-engine-examples/pom.xml +++ b/seatunnel-examples/seatunnel-engine-examples/pom.xml @@ -31,7 +31,18 @@ org.apache.seatunnel seatunnel-seatunnel-starter - ${revision} + ${project.version} + + + + org.apache.seatunnel + connector-fake + ${project.version} + + + org.apache.seatunnel + connector-console + ${project.version} \ No newline at end of file From 47988251c49bcf5187c3dd3e5487cbc5ca7deca3 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Tue, 6 Sep 2022 20:16:00 +0800 Subject: [PATCH 04/22] [Engine] [Core] Add seatunnel engine example --- .../java/org/apache/seatunnel/engine/server/SeaTunnelServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java index 56c00620625..eecd2f2f173 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java @@ -128,6 +128,7 @@ public void shutdown(boolean terminate) { if (resourceManager != null) { resourceManager.close(); } + executorService.shutdown(); taskExecutionService.shutdown(); } From ff0f019893d8d8fdbdfb7a66a67e16054693fddc Mon Sep 17 00:00:00 2001 From: Hisoka Date: Wed, 7 Sep 2022 15:29:24 +0800 Subject: [PATCH 05/22] [Engine] [Core] Add seatunnel engine starter --- seatunnel-core/pom.xml | 6 +- ... => start-seatunnel-flink-connector-v2.sh} | 0 .../core/starter/flink/FlinkStarter.java | 3 +- .../core/starter/flink/SeatunnelFlink.java | 3 +- .../starter/flink/utils/CommandLineUtils.java | 5 +- .../src/main/bin/start-seatunnel-seatunnel.sh | 64 ------------- .../starter/seatunnel/SeaTunnelStarter.java | 55 ----------- ... => start-seatunnel-spark-connector-v2.sh} | 0 .../core/starter/spark/SparkStarter.java | 2 +- .../pom.xml | 12 ++- .../src/main/bin/seatunnel.sh} | 17 +--- .../starter/seatunnel/CommandLineUtils.java | 19 +++- .../core/starter/seatunnel/SeaTunnel.java | 33 +++++++ .../args/ExecutionModeConverter.java | 34 +++++++ .../seatunnel/args/SeaTunnelCommandArgs.java | 30 +++++- .../SeaTunnelApiConfValidateCommand.java | 52 +++++++++++ .../SeaTunnelApiTaskExecuteCommand.java | 92 +++++++++++++++++++ .../command/SeaTunnelCommandBuilder.java | 33 +++++++ .../config/SeaTunnelApiConfigChecker.java | 31 +++++++ .../seatunnel/constant/SeaTunnelConstant.java | 22 +++++ .../src/main/assembly/assembly-bin-ci.xml | 15 +-- .../src/main/assembly/assembly-bin.xml | 15 +-- 22 files changed, 374 insertions(+), 169 deletions(-) rename seatunnel-core/seatunnel-flink-starter/src/main/bin/{start-seatunnel-flink-new-connector.sh => start-seatunnel-flink-connector-v2.sh} (100%) delete mode 100755 seatunnel-core/seatunnel-seatunnel-starter/src/main/bin/start-seatunnel-seatunnel.sh delete mode 100644 seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelStarter.java rename seatunnel-core/seatunnel-spark-starter/src/main/bin/{start-seatunnel-spark-new-connector.sh => start-seatunnel-spark-connector-v2.sh} (100%) rename seatunnel-core/{seatunnel-seatunnel-starter => seatunnel-starter}/pom.xml (85%) rename seatunnel-core/{seatunnel-seatunnel-starter/src/main/bin/start-seatunnel-engine-job.sh => seatunnel-starter/src/main/bin/seatunnel.sh} (76%) rename seatunnel-core/{seatunnel-seatunnel-starter => seatunnel-starter}/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java (65%) create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java rename seatunnel-core/{seatunnel-seatunnel-starter => seatunnel-starter}/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java (67%) create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/constant/SeaTunnelConstant.java diff --git a/seatunnel-core/pom.xml b/seatunnel-core/pom.xml index 9735a18e450..b5a588a83b8 100644 --- a/seatunnel-core/pom.xml +++ b/seatunnel-core/pom.xml @@ -44,7 +44,7 @@ seatunnel-core-starter seatunnel-flink-starter seatunnel-spark-starter - seatunnel-seatunnel-starter + seatunnel-starter @@ -52,7 +52,7 @@ seatunnel-core-base seatunnel-core-starter - seatunnel-seatunnel-starter + seatunnel-starter @@ -65,7 +65,7 @@ seatunnel-core-starter seatunnel-flink-starter seatunnel-spark-starter - seatunnel-seatunnel-starter + seatunnel-starter diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/bin/start-seatunnel-flink-new-connector.sh b/seatunnel-core/seatunnel-flink-starter/src/main/bin/start-seatunnel-flink-connector-v2.sh similarity index 100% rename from seatunnel-core/seatunnel-flink-starter/src/main/bin/start-seatunnel-flink-new-connector.sh rename to seatunnel-core/seatunnel-flink-starter/src/main/bin/start-seatunnel-flink-connector-v2.sh diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/FlinkStarter.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/FlinkStarter.java index 52b207a7943..4de645699ab 100644 --- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/FlinkStarter.java +++ b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/FlinkStarter.java @@ -20,7 +20,6 @@ import org.apache.seatunnel.common.config.Common; import org.apache.seatunnel.core.starter.Starter; import org.apache.seatunnel.core.starter.flink.args.FlinkCommandArgs; -import org.apache.seatunnel.core.starter.flink.config.FlinkJobType; import org.apache.seatunnel.core.starter.flink.utils.CommandLineUtils; import java.util.List; @@ -44,7 +43,7 @@ public class FlinkStarter implements Starter { private final String appJar; FlinkStarter(String[] args) { - this.flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.JAR); + this.flinkCommandArgs = CommandLineUtils.parseCommandArgs(args); // set the deployment mode, used to get the job jar path. Common.setDeployMode(flinkCommandArgs.getDeployMode()); Common.setStarter(true); diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/SeatunnelFlink.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/SeatunnelFlink.java index 330c2e501a7..20824a901e3 100644 --- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/SeatunnelFlink.java +++ b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/SeatunnelFlink.java @@ -22,13 +22,12 @@ import org.apache.seatunnel.core.starter.exception.CommandException; import org.apache.seatunnel.core.starter.flink.args.FlinkCommandArgs; import org.apache.seatunnel.core.starter.flink.command.FlinkCommandBuilder; -import org.apache.seatunnel.core.starter.flink.config.FlinkJobType; import org.apache.seatunnel.core.starter.flink.utils.CommandLineUtils; public class SeatunnelFlink { public static void main(String[] args) throws CommandException { - FlinkCommandArgs flinkCommandArgs = CommandLineUtils.parseCommandArgs(args, FlinkJobType.JAR); + FlinkCommandArgs flinkCommandArgs = CommandLineUtils.parseCommandArgs(args); Command flinkCommand = new FlinkCommandBuilder() .buildCommand(flinkCommandArgs); Seatunnel.run(flinkCommand); diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/utils/CommandLineUtils.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/utils/CommandLineUtils.java index 9b29fdf16d1..9d9a6a43ece 100644 --- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/utils/CommandLineUtils.java +++ b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/utils/CommandLineUtils.java @@ -18,7 +18,6 @@ package org.apache.seatunnel.core.starter.flink.utils; import org.apache.seatunnel.core.starter.flink.args.FlinkCommandArgs; -import org.apache.seatunnel.core.starter.flink.config.FlinkJobType; import org.apache.seatunnel.core.starter.flink.constant.FlinkConstant; import com.beust.jcommander.JCommander; @@ -43,10 +42,10 @@ public static FlinkCommandArgs parseFlinkArgs(String[] args) { return flinkCommandArgs; } - public static FlinkCommandArgs parseCommandArgs(String[] args, FlinkJobType jobType) { + public static FlinkCommandArgs parseCommandArgs(String[] args) { FlinkCommandArgs flinkCommandArgs = new FlinkCommandArgs(); JCommander jCommander = JCommander.newBuilder() - .programName(jobType.getType()) + .programName("start-seatunnel-flink-connector-v2.sh") .addObject(flinkCommandArgs) .acceptUnknownOptions(true) .args(args) diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/bin/start-seatunnel-seatunnel.sh b/seatunnel-core/seatunnel-seatunnel-starter/src/main/bin/start-seatunnel-seatunnel.sh deleted file mode 100755 index 1cfc741d83d..00000000000 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/bin/start-seatunnel-seatunnel.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -eu -# resolve links - $0 may be a softlink -PRG="$0" - -while [ -h "$PRG" ] ; do - # shellcheck disable=SC2006 - ls=`ls -ld "$PRG"` - # shellcheck disable=SC2006 - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - # shellcheck disable=SC2006 - PRG=`dirname "$PRG"`/"$link" - fi -done - -PRG_DIR=`dirname "$PRG"` -APP_DIR=`cd "$PRG_DIR/.." >/dev/null; pwd` -CONF_DIR=${APP_DIR}/config -APP_JAR=${APP_DIR}/lib/seatunnel-seatunnel-starter.jar -APP_MAIN="org.apache.seatunnel.core.starter.seatunnel.SeaTunnelStarter" - -if [ -f "${CONF_DIR}/seatunnel-env.sh" ]; then - . "${CONF_DIR}/seatunnel-env.sh" -fi - -if [ $# == 0 ] -then - args="-h" -else - args=$@ -fi - -CMD=$(java -cp ${APP_JAR} ${APP_MAIN} ${args}) && EXIT_CODE=$? || EXIT_CODE=$? -if [ ${EXIT_CODE} -eq 234 ]; then - # print usage - echo "${CMD}" - exit 0 -elif [ ${EXIT_CODE} -eq 0 ]; then - echo "Execute SeaTunnel Job: ${CMD}" - eval ${CMD} -else - echo "${CMD}" - exit ${EXIT_CODE} -fi diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelStarter.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelStarter.java deleted file mode 100644 index 94335cca925..00000000000 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelStarter.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.core.starter.seatunnel; - -import org.apache.seatunnel.common.config.Common; -import org.apache.seatunnel.common.config.DeployMode; -import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; -import org.apache.seatunnel.core.starter.utils.FileUtils; -import org.apache.seatunnel.engine.client.SeaTunnelClient; -import org.apache.seatunnel.engine.client.job.ClientJobProxy; -import org.apache.seatunnel.engine.client.job.JobExecutionEnvironment; -import org.apache.seatunnel.engine.common.config.ConfigProvider; -import org.apache.seatunnel.engine.common.config.JobConfig; - -import com.hazelcast.client.config.ClientConfig; - -import java.nio.file.Path; -import java.util.concurrent.ExecutionException; - -public class SeaTunnelStarter { - public static void main(String[] args) { - SeaTunnelCommandArgs seaTunnelCommandArgs = CommandLineUtils.parseSeaTunnelArgs(args); - Path configFile = FileUtils.getConfigPath(seaTunnelCommandArgs); - Common.setDeployMode(DeployMode.CLIENT); - JobConfig jobConfig = new JobConfig(); - jobConfig.setName("fake_to_file"); - - ClientConfig clientConfig = ConfigProvider.locateAndGetClientConfig(); - SeaTunnelClient engineClient = new SeaTunnelClient(clientConfig); - JobExecutionEnvironment jobExecutionEnv = engineClient.createExecutionContext(configFile.toString(), jobConfig); - - ClientJobProxy clientJobProxy; - try { - clientJobProxy = jobExecutionEnv.execute(); - clientJobProxy.waitForJobComplete(); - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException(e); - } - } -} diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-new-connector.sh b/seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-connector-v2.sh similarity index 100% rename from seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-new-connector.sh rename to seatunnel-core/seatunnel-spark-starter/src/main/bin/start-seatunnel-spark-connector-v2.sh diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java index 00261a80c03..e9cd2b41faf 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/SparkStarter.java @@ -134,7 +134,7 @@ static SparkStarter getInstance(String[] args) { private static SparkCommandArgs parseCommandArgs(String[] args) { SparkCommandArgs commandArgs = new SparkCommandArgs(); JCommander commander = JCommander.newBuilder() - .programName("start-seatunnel-spark.sh") + .programName("start-seatunnel-spark-connector-v2.sh") .addObject(commandArgs) .args(args) .build(); diff --git a/seatunnel-core/seatunnel-seatunnel-starter/pom.xml b/seatunnel-core/seatunnel-starter/pom.xml similarity index 85% rename from seatunnel-core/seatunnel-seatunnel-starter/pom.xml rename to seatunnel-core/seatunnel-starter/pom.xml index d2dd1ddbe63..81dccddaf82 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/pom.xml +++ b/seatunnel-core/seatunnel-starter/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - seatunnel-seatunnel-starter + seatunnel-starter @@ -46,6 +46,16 @@ seatunnel-engine-client ${project.version} + + + org.apache.seatunnel + seatunnel-engine-server + ${project.version} + + + ${project.name} + + \ No newline at end of file diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/bin/start-seatunnel-engine-job.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh similarity index 76% rename from seatunnel-core/seatunnel-seatunnel-starter/src/main/bin/start-seatunnel-engine-job.sh rename to seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh index 1cfc741d83d..b72c1f5fbcc 100755 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/bin/start-seatunnel-engine-job.sh +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh @@ -36,8 +36,8 @@ done PRG_DIR=`dirname "$PRG"` APP_DIR=`cd "$PRG_DIR/.." >/dev/null; pwd` CONF_DIR=${APP_DIR}/config -APP_JAR=${APP_DIR}/lib/seatunnel-seatunnel-starter.jar -APP_MAIN="org.apache.seatunnel.core.starter.seatunnel.SeaTunnelStarter" +APP_JAR=${APP_DIR}/lib/seatunnel-starter.jar +APP_MAIN="org.apache.seatunnel.core.starter.seatunnel.SeaTunnel" if [ -f "${CONF_DIR}/seatunnel-env.sh" ]; then . "${CONF_DIR}/seatunnel-env.sh" @@ -50,15 +50,4 @@ else args=$@ fi -CMD=$(java -cp ${APP_JAR} ${APP_MAIN} ${args}) && EXIT_CODE=$? || EXIT_CODE=$? -if [ ${EXIT_CODE} -eq 234 ]; then - # print usage - echo "${CMD}" - exit 0 -elif [ ${EXIT_CODE} -eq 0 ]; then - echo "Execute SeaTunnel Job: ${CMD}" - eval ${CMD} -else - echo "${CMD}" - exit ${EXIT_CODE} -fi +java -cp ${APP_JAR} ${APP_MAIN} ${args} diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java similarity index 65% rename from seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java rename to seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java index fd204e87a33..4e7d566cf91 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java @@ -18,21 +18,34 @@ package org.apache.seatunnel.core.starter.seatunnel; import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.constant.SeaTunnelConstant; import com.beust.jcommander.JCommander; +import com.beust.jcommander.UnixStyleUsageFormatter; public class CommandLineUtils { + private static final String SHELL_NAME = "seatunnel.sh"; + private CommandLineUtils() { throw new UnsupportedOperationException("CommandLineUtils is a utility class and cannot be instantiated"); } public static SeaTunnelCommandArgs parseSeaTunnelArgs(String[] args) { SeaTunnelCommandArgs seatunnelCommandArgs = new SeaTunnelCommandArgs(); - JCommander.newBuilder() + JCommander jCommander = JCommander.newBuilder() + .programName(SHELL_NAME) .addObject(seatunnelCommandArgs) - .build() - .parse(args); + .acceptUnknownOptions(true) + .args(args) + .build(); + // The args is not belongs to seatunnel, add into flink params + seatunnelCommandArgs.setSeatunnelParams(jCommander.getUnknownOptions()); + if (seatunnelCommandArgs.isHelp()) { + jCommander.setUsageFormatter(new UnixStyleUsageFormatter(jCommander)); + jCommander.usage(); + System.exit(SeaTunnelConstant.USAGE_EXIT_CODE); + } return seatunnelCommandArgs; } } diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java new file mode 100644 index 00000000000..a3a98a7b184 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel; + +import org.apache.seatunnel.core.starter.Seatunnel; +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.exception.CommandException; +import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.command.SeaTunnelCommandBuilder; + +public class SeaTunnel { + public static void main(String[] args) throws CommandException { + SeaTunnelCommandArgs seaTunnelCommandArgs = CommandLineUtils.parseSeaTunnelArgs(args); + Command command = + new SeaTunnelCommandBuilder().buildCommand(seaTunnelCommandArgs); + Seatunnel.run(command); + } +} diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java new file mode 100644 index 00000000000..ee7da538b9f --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.args; + +import org.apache.seatunnel.engine.common.runtime.ExecutionMode; + +import com.beust.jcommander.IStringConverter; +import com.beust.jcommander.ParameterException; + +public class ExecutionModeConverter implements IStringConverter { + @Override + public ExecutionMode convert(String value) { + try { + return ExecutionMode.valueOf(value.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new ParameterException("execution-mode: " + value + " is not allowed."); + } + } +} diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java similarity index 67% rename from seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java rename to seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java index 19c5d8587c3..deeb470dcfc 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java @@ -20,16 +20,44 @@ import org.apache.seatunnel.common.config.DeployMode; import org.apache.seatunnel.core.starter.command.AbstractCommandArgs; import org.apache.seatunnel.core.starter.config.EngineType; +import org.apache.seatunnel.engine.common.runtime.ExecutionMode; + +import com.beust.jcommander.Parameter; import java.util.List; public class SeaTunnelCommandArgs extends AbstractCommandArgs { /** - * Undefined parameters parsed will be stored here as seatunnel engint command parameters. + * Undefined parameters parsed will be stored here as seatunnel engine command parameters. */ private List seatunnelParams; + @Parameter(names = {"-n", "--name"}, + description = "The name of job") + private String name = "seatunnel_job"; + + @Parameter(names = {"-e", "--deploy-mode"}, + description = "SeaTunnel deploy mode", + converter = ExecutionModeConverter.class) + private ExecutionMode executionMode = ExecutionMode.LOCAL; + + public ExecutionMode getExecutionMode() { + return executionMode; + } + + public void setExecutionMode(ExecutionMode executionMode) { + this.executionMode = executionMode; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + @Override public EngineType getEngineType() { return EngineType.SEATUNNEL; diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java new file mode 100644 index 00000000000..8036120ae44 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.command; + +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.config.ConfigBuilder; +import org.apache.seatunnel.core.starter.exception.ConfigCheckException; +import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.config.SeaTunnelApiConfigChecker; +import org.apache.seatunnel.core.starter.utils.FileUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; + +/** + * Use to validate the configuration of the SeaTunnel API. + */ +public class SeaTunnelApiConfValidateCommand implements Command { + + private static final Logger LOGGER = LoggerFactory.getLogger(SeaTunnelApiConfValidateCommand.class); + + private final SeaTunnelCommandArgs seaTunnelCommandArgs; + + public SeaTunnelApiConfValidateCommand(SeaTunnelCommandArgs seaTunnelCommandArgs) { + this.seaTunnelCommandArgs = seaTunnelCommandArgs; + } + + @Override + public void execute() throws ConfigCheckException { + Path configPath = FileUtils.getConfigPath(seaTunnelCommandArgs); + ConfigBuilder configBuilder = new ConfigBuilder(configPath); + new SeaTunnelApiConfigChecker().checkConfig(configBuilder.getConfig()); + LOGGER.info("config OK !"); + } +} diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java new file mode 100644 index 00000000000..7d74706ef37 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.command; + +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.exception.CommandExecuteException; +import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.utils.FileUtils; +import org.apache.seatunnel.engine.client.SeaTunnelClient; +import org.apache.seatunnel.engine.client.job.ClientJobProxy; +import org.apache.seatunnel.engine.client.job.JobExecutionEnvironment; +import org.apache.seatunnel.engine.common.config.ConfigProvider; +import org.apache.seatunnel.engine.common.config.JobConfig; +import org.apache.seatunnel.engine.common.config.SeaTunnelConfig; +import org.apache.seatunnel.engine.common.runtime.ExecutionMode; +import org.apache.seatunnel.engine.server.SeaTunnelNodeContext; + +import com.hazelcast.client.config.ClientConfig; +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.instance.impl.HazelcastInstanceFactory; + +import java.nio.file.Path; +import java.util.concurrent.ExecutionException; + +/** + * This command is used to execute the SeaTunnel engine job by SeaTunnel API. + */ +public class SeaTunnelApiTaskExecuteCommand implements Command { + + private final SeaTunnelCommandArgs seaTunnelCommandArgs; + + // TODO custom cluster name on cluster execution mode + private static final String CLUSTER_NAME = "SeaTunnelCluster"; + + public SeaTunnelApiTaskExecuteCommand(SeaTunnelCommandArgs seaTunnelCommandArgs) { + this.seaTunnelCommandArgs = seaTunnelCommandArgs; + } + + @Override + public void execute() throws CommandExecuteException { + Path configFile = FileUtils.getConfigPath(seaTunnelCommandArgs); + + JobConfig jobConfig = new JobConfig(); + jobConfig.setName(seaTunnelCommandArgs.getName()); + + HazelcastInstance instance = null; + if (seaTunnelCommandArgs.getExecutionMode().equals(ExecutionMode.LOCAL)) { + instance = createServerInLocal(); + } + + ClientConfig clientConfig = ConfigProvider.locateAndGetClientConfig(); + clientConfig.setClusterName(CLUSTER_NAME); + SeaTunnelClient engineClient = new SeaTunnelClient(clientConfig); + JobExecutionEnvironment jobExecutionEnv = engineClient.createExecutionContext(configFile.toString(), jobConfig); + + ClientJobProxy clientJobProxy; + try { + clientJobProxy = jobExecutionEnv.execute(); + clientJobProxy.waitForJobComplete(); + } catch (ExecutionException | InterruptedException e) { + throw new CommandExecuteException("SeaTunnel job executed failed", e); + } finally { + if (instance != null) { + instance.shutdown(); + } + } + } + + private HazelcastInstance createServerInLocal() { + SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); + seaTunnelConfig.getHazelcastConfig().setClusterName(CLUSTER_NAME); + return HazelcastInstanceFactory.newHazelcastInstance(seaTunnelConfig.getHazelcastConfig(), + Thread.currentThread().getName(), + new SeaTunnelNodeContext(ConfigProvider.locateAndGetSeaTunnelConfig())); + } + +} diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java new file mode 100644 index 00000000000..722c84e7b0d --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.command; + +import org.apache.seatunnel.common.config.Common; +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.command.CommandBuilder; +import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; + +public class SeaTunnelCommandBuilder implements CommandBuilder { + + @Override + public Command buildCommand(SeaTunnelCommandArgs commandArgs) { + Common.setDeployMode(commandArgs.getDeployMode()); + return commandArgs.isCheckConfig() ? new SeaTunnelApiConfValidateCommand(commandArgs) + : new SeaTunnelApiTaskExecuteCommand(commandArgs); + } +} diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java new file mode 100644 index 00000000000..bfaff2a9c35 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.config; + +import org.apache.seatunnel.core.starter.config.ConfigChecker; +import org.apache.seatunnel.core.starter.exception.ConfigCheckException; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +public class SeaTunnelApiConfigChecker implements ConfigChecker { + + @Override + public void checkConfig(Config config) throws ConfigCheckException { + // TODO implement + } +} diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/constant/SeaTunnelConstant.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/constant/SeaTunnelConstant.java new file mode 100644 index 00000000000..7b032b9f442 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/constant/SeaTunnelConstant.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.constant; + +public class SeaTunnelConstant { + public static final int USAGE_EXIT_CODE = 234; +} diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml index d4f7aaffa6e..87eb3a111d0 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml @@ -120,19 +120,14 @@ - ../seatunnel-engine/seatunnel-engine-client/target - - seatunnel-engine-*.jar - - - %regex[.*((javadoc)|(sources))\.jar] - - /lib + ../seatunnel-core/seatunnel-starter/src/main/bin + /bin + 0755 - ../seatunnel-engine/seatunnel-engine-server/target + ../seatunnel-core/seatunnel-starter/target - seatunnel-engine-*.jar + seatunnel-starter*.jar %regex[.*((javadoc)|(sources))\.jar] diff --git a/seatunnel-dist/src/main/assembly/assembly-bin.xml b/seatunnel-dist/src/main/assembly/assembly-bin.xml index eddaf812967..6da678c6e7f 100644 --- a/seatunnel-dist/src/main/assembly/assembly-bin.xml +++ b/seatunnel-dist/src/main/assembly/assembly-bin.xml @@ -120,19 +120,14 @@ - ../seatunnel-engine/seatunnel-engine-client/target - - seatunnel-engine-*.jar - - - %regex[.*((javadoc)|(sources))\.jar] - - /lib + ../seatunnel-core/seatunnel-starter/src/main/bin + /bin + 0755 - ../seatunnel-engine/seatunnel-engine-server/target + ../seatunnel-core/seatunnel-starter/target - seatunnel-engine-*.jar + seatunnel-starter*.jar %regex[.*((javadoc)|(sources))\.jar] From 2531fd2044aac71d2c5cc1278816843c8fc153d3 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Wed, 7 Sep 2022 15:50:10 +0800 Subject: [PATCH 06/22] [Engine] [Example] Add random cluster name when use local mode --- .../seatunnel/args/SeaTunnelCommandArgs.java | 12 ++++++++++++ .../SeaTunnelApiTaskExecuteCommand.java | 18 +++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java index deeb470dcfc..9e86b280005 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java @@ -37,6 +37,10 @@ public class SeaTunnelCommandArgs extends AbstractCommandArgs { description = "The name of job") private String name = "seatunnel_job"; + @Parameter(names = {"-cn", "--cluster"}, + description = "The name of cluster") + private String clusterName = "seatunnel_default_cluster"; + @Parameter(names = {"-e", "--deploy-mode"}, description = "SeaTunnel deploy mode", converter = ExecutionModeConverter.class) @@ -58,6 +62,14 @@ public void setName(String name) { this.name = name; } + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + @Override public EngineType getEngineType() { return EngineType.SEATUNNEL; diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java index 7d74706ef37..7086d0ce705 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java @@ -35,6 +35,7 @@ import com.hazelcast.instance.impl.HazelcastInstanceFactory; import java.nio.file.Path; +import java.util.Random; import java.util.concurrent.ExecutionException; /** @@ -45,7 +46,6 @@ public class SeaTunnelApiTaskExecuteCommand implements Command Date: Wed, 7 Sep 2022 18:36:27 +0800 Subject: [PATCH 07/22] [Engine] [Starter] Remove SeaTunnel engine Starter spark/flink dependency and bug fixed. --- pom.xml | 7 ++ seatunnel-apis/seatunnel-api-base/pom.xml | 4 - seatunnel-core/README.md | 5 +- seatunnel-core/seatunnel-core-starter/pom.xml | 10 +- .../config/AbstractExecutionContext.java | 92 ------------------- .../starter/config/EnvironmentFactory.java | 25 ++--- .../core/starter/config/ExecutionFactory.java | 84 ----------------- .../seatunnel-flink-starter/pom.xml | 6 -- .../flink/config/FlinkApiConfigChecker.java | 3 +- .../flink/config/FlinkApiEnvironment.java | 75 --------------- .../flink/config/FlinkEnvironmentFactory.java | 35 +++++++ .../flink/execution/FlinkExecution.java | 7 +- .../seatunnel-spark-starter/pom.xml | 6 -- .../spark/config/SparkApiConfigChecker.java | 1 + ...ment.java => SparkEnvironmentFactory.java} | 46 ++-------- .../spark/execution/SparkExecution.java | 7 +- .../engine/client/job/JobConfigParser.java | 2 +- 17 files changed, 72 insertions(+), 343 deletions(-) delete mode 100644 seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/AbstractExecutionContext.java delete mode 100644 seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/ExecutionFactory.java delete mode 100644 seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiEnvironment.java create mode 100644 seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkEnvironmentFactory.java rename seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/{SparkEnvironment.java => SparkEnvironmentFactory.java} (53%) diff --git a/pom.xml b/pom.xml index 9750e140ea0..5f53261817b 100644 --- a/pom.xml +++ b/pom.xml @@ -197,6 +197,7 @@ 2.11.0 4.4 3.3.0 + 1.20 1.8.0 provided provided @@ -629,6 +630,12 @@ ${jackson.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + + com.fasterxml.jackson.module jackson-module-scala_${scala.binary.version} diff --git a/seatunnel-apis/seatunnel-api-base/pom.xml b/seatunnel-apis/seatunnel-api-base/pom.xml index 9fe328718ae..502cae6c41f 100644 --- a/seatunnel-apis/seatunnel-api-base/pom.xml +++ b/seatunnel-apis/seatunnel-api-base/pom.xml @@ -36,10 +36,6 @@ ${project.version} - - org.apache.seatunnel - seatunnel-config-shade - diff --git a/seatunnel-core/README.md b/seatunnel-core/README.md index c4b48e4cb4e..05fdf0e15e1 100644 --- a/seatunnel-core/README.md +++ b/seatunnel-core/README.md @@ -5,4 +5,7 @@ This module is the seatunnel job entrypoint. Seatunnel jobs are started by the b - seatunnel-core-flink: The flink job starter. - seatunnel-core-flink-sql: The flink sql job starter. -- seatunnel-core-spark: The spark job starter. \ No newline at end of file +- seatunnel-core-spark: The spark job starter. +- seatunnel-spark-starter: The spark job starter for connector-v2. +- seatunnel-spark-starter: The flink job starter for connector-v2. +- seatunnel-starter: The seatunnel engine job starter for connector-v2. \ No newline at end of file diff --git a/seatunnel-core/seatunnel-core-starter/pom.xml b/seatunnel-core/seatunnel-core-starter/pom.xml index 561879c6826..8d05a054566 100644 --- a/seatunnel-core/seatunnel-core-starter/pom.xml +++ b/seatunnel-core/seatunnel-core-starter/pom.xml @@ -40,19 +40,19 @@ org.apache.seatunnel - seatunnel-api-flink + seatunnel-api ${project.version} org.apache.seatunnel - seatunnel-api-spark + seatunnel-plugin-discovery ${project.version} + - org.apache.seatunnel - seatunnel-plugin-discovery - ${project.version} + org.apache.commons + commons-compress diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/AbstractExecutionContext.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/AbstractExecutionContext.java deleted file mode 100644 index a834965c099..00000000000 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/AbstractExecutionContext.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.core.starter.config; - -import org.apache.seatunnel.apis.base.api.BaseSink; -import org.apache.seatunnel.apis.base.api.BaseSource; -import org.apache.seatunnel.apis.base.api.BaseTransform; -import org.apache.seatunnel.apis.base.env.RuntimeEnv; -import org.apache.seatunnel.common.constants.JobMode; -import org.apache.seatunnel.common.constants.PluginType; -import org.apache.seatunnel.plugin.discovery.PluginIdentifier; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import java.net.URL; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * The ExecutionContext contains all configuration needed to run the job. - * - * @param environment type. - */ -public abstract class AbstractExecutionContext { - - private final Config config; - private final EngineType engine; - - private final ENVIRONMENT environment; - private final JobMode jobMode; - - public AbstractExecutionContext(Config config, EngineType engine) { - this.config = config; - this.engine = engine; - this.environment = new EnvironmentFactory(config, engine).getEnvironment(); - this.jobMode = environment.getJobMode(); - } - - public Config getRootConfig() { - return config; - } - - public EngineType getEngine() { - return engine; - } - - public ENVIRONMENT getEnvironment() { - return environment; - } - - public JobMode getJobMode() { - return jobMode; - } - - public abstract List> getSources(); - - public abstract List> getTransforms(); - - public abstract List> getSinks(); - - public abstract List getPluginJars(); - - @SuppressWarnings("checkstyle:Indentation") - protected List getPluginIdentifiers(PluginType... pluginTypes) { - return Arrays.stream(pluginTypes).flatMap((Function>) pluginType -> { - List configList = config.getConfigList(pluginType.getType()); - return configList.stream() - .map(pluginConfig -> PluginIdentifier - .of(engine.getEngine(), - pluginType.getType(), - pluginConfig.getString("plugin_name"))); - }).collect(Collectors.toList()); - } -} diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/EnvironmentFactory.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/EnvironmentFactory.java index 20e77a1dbb1..5b99b7aa0c3 100644 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/EnvironmentFactory.java +++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/EnvironmentFactory.java @@ -19,8 +19,6 @@ import org.apache.seatunnel.apis.base.env.RuntimeEnv; import org.apache.seatunnel.common.constants.JobMode; -import org.apache.seatunnel.flink.FlinkEnvironment; -import org.apache.seatunnel.spark.SparkEnvironment; import org.apache.seatunnel.shade.com.typesafe.config.Config; @@ -31,39 +29,28 @@ * * @param environment type */ -public class EnvironmentFactory { +public abstract class EnvironmentFactory { private static final String PLUGIN_NAME_KEY = "plugin_name"; private final Config config; - private final EngineType engine; - public EnvironmentFactory(Config config, EngineType engine) { + public EnvironmentFactory(Config config) { this.config = config; - this.engine = engine; } // todo:put this method into submodule to avoid dependency on the engine public synchronized ENVIRONMENT getEnvironment() { Config envConfig = config.getConfig("env"); - boolean enableHive = checkIsContainHive(); - ENVIRONMENT env; - switch (engine) { - case SPARK: - env = (ENVIRONMENT) new SparkEnvironment().setEnableHive(enableHive); - break; - case FLINK: - env = (ENVIRONMENT) new FlinkEnvironment(); - break; - default: - throw new IllegalArgumentException("Engine: " + engine + " is not supported"); - } + ENVIRONMENT env = newEnvironment(); env.setConfig(envConfig) .setJobMode(getJobMode(envConfig)).prepare(); return env; } - private boolean checkIsContainHive() { + protected abstract ENVIRONMENT newEnvironment(); + + protected boolean checkIsContainHive() { List sourceConfigList = config.getConfigList(PluginType.SOURCE.getType()); for (Config c : sourceConfigList) { if (c.getString(PLUGIN_NAME_KEY).toLowerCase().contains("hive")) { diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/ExecutionFactory.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/ExecutionFactory.java deleted file mode 100644 index d0d99cd948b..00000000000 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/config/ExecutionFactory.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.core.starter.config; - -import org.apache.seatunnel.apis.base.api.BaseSink; -import org.apache.seatunnel.apis.base.api.BaseSource; -import org.apache.seatunnel.apis.base.api.BaseTransform; -import org.apache.seatunnel.apis.base.env.Execution; -import org.apache.seatunnel.apis.base.env.RuntimeEnv; -import org.apache.seatunnel.flink.FlinkEnvironment; -import org.apache.seatunnel.flink.batch.FlinkBatchExecution; -import org.apache.seatunnel.flink.stream.FlinkStreamExecution; -import org.apache.seatunnel.spark.SparkEnvironment; -import org.apache.seatunnel.spark.batch.SparkBatchExecution; -import org.apache.seatunnel.spark.stream.SparkStreamingExecution; -import org.apache.seatunnel.spark.structuredstream.StructuredStreamingExecution; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Used to create {@link Execution}. - * - * @param environment type - */ -public class ExecutionFactory { - - private static final Logger LOGGER = LoggerFactory.getLogger(ExecutionFactory.class); - - public AbstractExecutionContext executionContext; - - public ExecutionFactory(AbstractExecutionContext executionContext) { - this.executionContext = executionContext; - } - - public Execution, BaseTransform, BaseSink, ENVIRONMENT> createExecution() { - Execution execution = null; - switch (executionContext.getEngine()) { - case SPARK: - SparkEnvironment sparkEnvironment = (SparkEnvironment) executionContext.getEnvironment(); - switch (executionContext.getJobMode()) { - case STREAMING: - execution = new SparkStreamingExecution(sparkEnvironment); - break; - case STRUCTURED_STREAMING: - execution = new StructuredStreamingExecution(sparkEnvironment); - break; - default: - execution = new SparkBatchExecution(sparkEnvironment); - } - break; - case FLINK: - FlinkEnvironment flinkEnvironment = (FlinkEnvironment) executionContext.getEnvironment(); - switch (executionContext.getJobMode()) { - case STREAMING: - execution = new FlinkStreamExecution(flinkEnvironment); - break; - default: - execution = new FlinkBatchExecution(flinkEnvironment); - } - break; - default: - throw new IllegalArgumentException("No suitable engine"); - } - LOGGER.info("current execution is [{}]", execution.getClass().getName()); - return (Execution, BaseTransform, BaseSink, ENVIRONMENT>) execution; - } - -} diff --git a/seatunnel-core/seatunnel-flink-starter/pom.xml b/seatunnel-core/seatunnel-flink-starter/pom.xml index 6385b9c41f6..15131900b87 100644 --- a/seatunnel-core/seatunnel-flink-starter/pom.xml +++ b/seatunnel-core/seatunnel-flink-starter/pom.xml @@ -42,12 +42,6 @@ ${project.version} - - org.apache.seatunnel - seatunnel-api - ${project.version} - - org.apache.seatunnel seatunnel-translation-flink diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiConfigChecker.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiConfigChecker.java index 2ae68515d0a..81058db280d 100644 --- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiConfigChecker.java +++ b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiConfigChecker.java @@ -19,10 +19,11 @@ import org.apache.seatunnel.core.starter.config.ConfigChecker; import org.apache.seatunnel.core.starter.exception.ConfigCheckException; +import org.apache.seatunnel.flink.FlinkEnvironment; import org.apache.seatunnel.shade.com.typesafe.config.Config; -public class FlinkApiConfigChecker implements ConfigChecker { +public class FlinkApiConfigChecker implements ConfigChecker { @Override public void checkConfig(Config config) throws ConfigCheckException { diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiEnvironment.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiEnvironment.java deleted file mode 100644 index 5b8ba32d89d..00000000000 --- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkApiEnvironment.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.core.starter.flink.config; - -import org.apache.seatunnel.apis.base.env.RuntimeEnv; -import org.apache.seatunnel.common.config.CheckResult; -import org.apache.seatunnel.common.constants.JobMode; - -import org.apache.seatunnel.shade.com.typesafe.config.Config; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URL; -import java.util.List; - -public class FlinkApiEnvironment implements RuntimeEnv { - - private static final Logger LOGGER = LoggerFactory.getLogger(FlinkApiEnvironment.class); - - private Config config; - - @Override - public FlinkApiEnvironment setConfig(Config config) { - this.config = config; - return this; - } - - @Override - public Config getConfig() { - return config; - } - - @Override - public CheckResult checkConfig() { - // todo - return null; - } - - @Override - public FlinkApiEnvironment prepare() { - // todo - return null; - } - - @Override - public FlinkApiEnvironment setJobMode(JobMode mode) { - return null; - } - - @Override - public JobMode getJobMode() { - return null; - } - - @Override - public void registerPlugin(List pluginPaths) { - - } -} diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkEnvironmentFactory.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkEnvironmentFactory.java new file mode 100644 index 00000000000..a3d3c2eec9a --- /dev/null +++ b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/config/FlinkEnvironmentFactory.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.flink.config; + +import org.apache.seatunnel.core.starter.config.EnvironmentFactory; +import org.apache.seatunnel.flink.FlinkEnvironment; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +public class FlinkEnvironmentFactory extends EnvironmentFactory { + + public FlinkEnvironmentFactory(Config config) { + super(config); + } + + @Override + protected FlinkEnvironment newEnvironment() { + return new FlinkEnvironment(); + } +} diff --git a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkExecution.java b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkExecution.java index c0cc1a67e38..be42cb36d9d 100644 --- a/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkExecution.java +++ b/seatunnel-core/seatunnel-flink-starter/src/main/java/org/apache/seatunnel/core/starter/flink/execution/FlinkExecution.java @@ -18,10 +18,9 @@ package org.apache.seatunnel.core.starter.flink.execution; import org.apache.seatunnel.api.common.SeaTunnelContext; -import org.apache.seatunnel.core.starter.config.EngineType; -import org.apache.seatunnel.core.starter.config.EnvironmentFactory; import org.apache.seatunnel.core.starter.exception.TaskExecuteException; import org.apache.seatunnel.core.starter.execution.TaskExecution; +import org.apache.seatunnel.core.starter.flink.config.FlinkEnvironmentFactory; import org.apache.seatunnel.flink.FlinkEnvironment; import org.apache.seatunnel.shade.com.typesafe.config.Config; @@ -41,15 +40,13 @@ public class FlinkExecution implements TaskExecution { private static final Logger LOGGER = LoggerFactory.getLogger(FlinkExecution.class); - private final Config config; private final FlinkEnvironment flinkEnvironment; private final PluginExecuteProcessor sourcePluginExecuteProcessor; private final PluginExecuteProcessor transformPluginExecuteProcessor; private final PluginExecuteProcessor sinkPluginExecuteProcessor; public FlinkExecution(Config config) { - this.config = config; - this.flinkEnvironment = new EnvironmentFactory(config, EngineType.FLINK).getEnvironment(); + this.flinkEnvironment = new FlinkEnvironmentFactory(config).getEnvironment(); SeaTunnelContext.getContext().setJobMode(flinkEnvironment.getJobMode()); this.sourcePluginExecuteProcessor = new SourceExecuteProcessor(flinkEnvironment, config.getConfigList("source")); this.transformPluginExecuteProcessor = new TransformExecuteProcessor(flinkEnvironment, config.getConfigList("transform")); diff --git a/seatunnel-core/seatunnel-spark-starter/pom.xml b/seatunnel-core/seatunnel-spark-starter/pom.xml index d65b58739f7..e02470ea2d7 100644 --- a/seatunnel-core/seatunnel-spark-starter/pom.xml +++ b/seatunnel-core/seatunnel-spark-starter/pom.xml @@ -35,12 +35,6 @@ - - org.apache.seatunnel - seatunnel-api - ${project.version} - - org.apache.seatunnel seatunnel-core-starter diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java index b08a4a752c2..3f340edafd5 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.core.starter.config.ConfigChecker; import org.apache.seatunnel.core.starter.exception.ConfigCheckException; +import org.apache.seatunnel.spark.SparkEnvironment; import org.apache.seatunnel.shade.com.typesafe.config.Config; diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkEnvironment.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkEnvironmentFactory.java similarity index 53% rename from seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkEnvironment.java rename to seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkEnvironmentFactory.java index f7d5b83d4a4..8ac703ea8e8 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkEnvironment.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkEnvironmentFactory.java @@ -1,4 +1,3 @@ - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -18,49 +17,18 @@ package org.apache.seatunnel.core.starter.spark.config; -import org.apache.seatunnel.apis.base.env.RuntimeEnv; -import org.apache.seatunnel.common.config.CheckResult; -import org.apache.seatunnel.common.constants.JobMode; +import org.apache.seatunnel.core.starter.config.EnvironmentFactory; +import org.apache.seatunnel.spark.SparkEnvironment; import org.apache.seatunnel.shade.com.typesafe.config.Config; -import java.net.URL; -import java.util.List; - -public class SparkEnvironment implements RuntimeEnv { - - @Override - public SparkEnvironment setConfig(Config config) { - return null; - } - - @Override - public Config getConfig() { - return null; - } - - @Override - public CheckResult checkConfig() { - return null; - } - - @Override - public SparkEnvironment prepare() { - return null; - } - - @Override - public SparkEnvironment setJobMode(JobMode mode) { - return null; +public class SparkEnvironmentFactory extends EnvironmentFactory { + public SparkEnvironmentFactory(Config config) { + super(config); } @Override - public JobMode getJobMode() { - return null; - } - - @Override - public void registerPlugin(List pluginPaths) { - + protected SparkEnvironment newEnvironment() { + return new SparkEnvironment().setEnableHive(checkIsContainHive()); } } diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java index 7b90bd0dd13..612e017e582 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java @@ -18,9 +18,8 @@ package org.apache.seatunnel.core.starter.spark.execution; import org.apache.seatunnel.api.common.SeaTunnelContext; -import org.apache.seatunnel.core.starter.config.EngineType; -import org.apache.seatunnel.core.starter.config.EnvironmentFactory; import org.apache.seatunnel.core.starter.exception.TaskExecuteException; +import org.apache.seatunnel.core.starter.spark.config.SparkEnvironmentFactory; import org.apache.seatunnel.spark.SparkEnvironment; import org.apache.seatunnel.shade.com.typesafe.config.Config; @@ -37,15 +36,13 @@ public class SparkExecution { private static final Logger LOGGER = LoggerFactory.getLogger(SparkExecution.class); - private final Config config; private final SparkEnvironment sparkEnvironment; private final PluginExecuteProcessor sourcePluginExecuteProcessor; private final PluginExecuteProcessor transformPluginExecuteProcessor; private final PluginExecuteProcessor sinkPluginExecuteProcessor; public SparkExecution(Config config) { - this.config = config; - this.sparkEnvironment = (SparkEnvironment) new EnvironmentFactory<>(config, EngineType.SPARK).getEnvironment(); + this.sparkEnvironment = new SparkEnvironmentFactory(config).getEnvironment(); SeaTunnelContext.getContext().setJobMode(sparkEnvironment.getJobMode()); this.sourcePluginExecuteProcessor = new SourceExecuteProcessor(sparkEnvironment, config.getConfigList("source")); this.transformPluginExecuteProcessor = new TransformExecuteProcessor(sparkEnvironment, config.getConfigList("transform")); diff --git a/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/JobConfigParser.java b/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/JobConfigParser.java index 964bc5593d0..14c0df4f59a 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/JobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/JobConfigParser.java @@ -28,7 +28,7 @@ import org.apache.seatunnel.apis.base.plugin.Plugin; import org.apache.seatunnel.common.constants.CollectionConstants; import org.apache.seatunnel.common.constants.JobMode; -import org.apache.seatunnel.core.base.config.ConfigBuilder; +import org.apache.seatunnel.core.starter.config.ConfigBuilder; import org.apache.seatunnel.engine.common.config.JobConfig; import org.apache.seatunnel.engine.common.exception.JobDefineCheckException; import org.apache.seatunnel.engine.common.utils.IdGenerator; From 015118a0c8599a5f9e2ff701fdff09c25d96ada8 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Wed, 7 Sep 2022 19:14:01 +0800 Subject: [PATCH 08/22] [Engine] [Example] Fix review --- .../SeaTunnelApiTaskExecuteCommand.java | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java index 7086d0ce705..d9d8638d9a7 100644 --- a/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java @@ -57,21 +57,19 @@ public void execute() throws CommandExecuteException { JobConfig jobConfig = new JobConfig(); jobConfig.setName(seaTunnelCommandArgs.getName()); - HazelcastInstance instance = null; - String clusterName = seaTunnelCommandArgs.getClusterName(); - if (seaTunnelCommandArgs.getExecutionMode().equals(ExecutionMode.LOCAL)) { - clusterName = creatRandomClusterName(clusterName); - instance = createServerInLocal(clusterName); - } - - ClientConfig clientConfig = ConfigProvider.locateAndGetClientConfig(); - clientConfig.setClusterName(clusterName); - SeaTunnelClient engineClient = new SeaTunnelClient(clientConfig); - JobExecutionEnvironment jobExecutionEnv = engineClient.createExecutionContext(configFile.toString(), jobConfig); - - ClientJobProxy clientJobProxy; try { + String clusterName = seaTunnelCommandArgs.getClusterName(); + if (seaTunnelCommandArgs.getExecutionMode().equals(ExecutionMode.LOCAL)) { + clusterName = creatRandomClusterName(clusterName); + instance = createServerInLocal(clusterName); + } + ClientConfig clientConfig = ConfigProvider.locateAndGetClientConfig(); + clientConfig.setClusterName(clusterName); + SeaTunnelClient engineClient = new SeaTunnelClient(clientConfig); + JobExecutionEnvironment jobExecutionEnv = engineClient.createExecutionContext(configFile.toString(), jobConfig); + + ClientJobProxy clientJobProxy; clientJobProxy = jobExecutionEnv.execute(); clientJobProxy.waitForJobComplete(); } catch (ExecutionException | InterruptedException e) { @@ -88,7 +86,7 @@ private HazelcastInstance createServerInLocal(String clusterName) { seaTunnelConfig.getHazelcastConfig().setClusterName(clusterName); return HazelcastInstanceFactory.newHazelcastInstance(seaTunnelConfig.getHazelcastConfig(), Thread.currentThread().getName(), - new SeaTunnelNodeContext(ConfigProvider.locateAndGetSeaTunnelConfig())); + new SeaTunnelNodeContext(seaTunnelConfig)); } @SuppressWarnings("checkstyle:MagicNumber") From eac73f99a65f6afe06d0e2a4ac64bb67215ef083 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Wed, 7 Sep 2022 20:57:16 +0800 Subject: [PATCH 09/22] [Engine] [Starter] Add seatunnel cluster and client entrance --- .../src/main/bin/seatunnel-cluster.sh | 53 +++++++++++++++ .../src/main/bin/seatunnel.sh | 2 +- .../starter/seatunnel/CommandLineUtils.java | 53 ++++++++++----- .../{SeaTunnel.java => SeaTunnelClient.java} | 12 ++-- .../starter/seatunnel/SeaTunnelServer.java | 33 ++++++++++ ...ommandArgs.java => ClientCommandArgs.java} | 51 ++++++--------- .../seatunnel/args/ServerCommandArgs.java | 65 +++++++++++++++++++ ...mmand.java => ApiConfValidateCommand.java} | 14 ++-- ...Builder.java => ClientCommandBuilder.java} | 10 +-- ...Command.java => ClientExecuteCommand.java} | 20 +++--- .../command/ServerCommandBuilder.java | 30 +++++++++ .../command/ServerExecuteCommand.java | 48 ++++++++++++++ .../seatunnel-engine-client/pom.xml | 4 +- .../src/test/resources/client_test.conf | 14 +--- .../seatunnel-engine-examples/pom.xml | 2 +- .../engine/SeaTunnelEngineExample.java | 16 ++--- 16 files changed, 327 insertions(+), 100 deletions(-) create mode 100755 seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh rename seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/{SeaTunnel.java => SeaTunnelClient.java} (73%) create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelServer.java rename seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/{SeaTunnelCommandArgs.java => ClientCommandArgs.java} (86%) create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ServerCommandArgs.java rename seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/{SeaTunnelApiConfValidateCommand.java => ApiConfValidateCommand.java} (77%) rename seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/{SeaTunnelCommandBuilder.java => ClientCommandBuilder.java} (73%) rename seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/{SeaTunnelApiTaskExecuteCommand.java => ClientExecuteCommand.java} (83%) create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerCommandBuilder.java create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommand.java diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh new file mode 100755 index 00000000000..debf019999e --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eu +# resolve links - $0 may be a softlink +PRG="$0" + +while [ -h "$PRG" ] ; do + # shellcheck disable=SC2006 + ls=`ls -ld "$PRG"` + # shellcheck disable=SC2006 + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + # shellcheck disable=SC2006 + PRG=`dirname "$PRG"`/"$link" + fi +done + +PRG_DIR=`dirname "$PRG"` +APP_DIR=`cd "$PRG_DIR/.." >/dev/null; pwd` +CONF_DIR=${APP_DIR}/config +APP_JAR=${APP_DIR}/lib/seatunnel-starter.jar +APP_MAIN="org.apache.seatunnel.core.starter.seatunnel.SeaTunnelServer" + +if [ -f "${CONF_DIR}/seatunnel-env.sh" ]; then + . "${CONF_DIR}/seatunnel-env.sh" +fi + +if [ $# == 0 ] +then + args="-h" +else + args=$@ +fi + +java -cp ${APP_JAR} ${APP_MAIN} ${args} diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh index b72c1f5fbcc..601bfd4e7ff 100755 --- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh @@ -37,7 +37,7 @@ PRG_DIR=`dirname "$PRG"` APP_DIR=`cd "$PRG_DIR/.." >/dev/null; pwd` CONF_DIR=${APP_DIR}/config APP_JAR=${APP_DIR}/lib/seatunnel-starter.jar -APP_MAIN="org.apache.seatunnel.core.starter.seatunnel.SeaTunnel" +APP_MAIN="org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient" if [ -f "${CONF_DIR}/seatunnel-env.sh" ]; then . "${CONF_DIR}/seatunnel-env.sh" diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java index 4e7d566cf91..5c0e1b65066 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/CommandLineUtils.java @@ -17,7 +17,8 @@ package org.apache.seatunnel.core.starter.seatunnel; -import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.args.ServerCommandArgs; import org.apache.seatunnel.core.starter.seatunnel.constant.SeaTunnelConstant; import com.beust.jcommander.JCommander; @@ -25,27 +26,49 @@ public class CommandLineUtils { - private static final String SHELL_NAME = "seatunnel.sh"; + private static final String CLIENT_SHELL_NAME = "seatunnel.sh"; + + private static final String SERVER_SHELL_NAME = "seatunnel.sh"; private CommandLineUtils() { throw new UnsupportedOperationException("CommandLineUtils is a utility class and cannot be instantiated"); } - public static SeaTunnelCommandArgs parseSeaTunnelArgs(String[] args) { - SeaTunnelCommandArgs seatunnelCommandArgs = new SeaTunnelCommandArgs(); - JCommander jCommander = JCommander.newBuilder() - .programName(SHELL_NAME) - .addObject(seatunnelCommandArgs) + public static ClientCommandArgs parseSeaTunnelClientArgs(String[] args) { + ClientCommandArgs clientCommandArgs = new ClientCommandArgs(); + JCommander jCommander = getJCommander(CLIENT_SHELL_NAME, args, clientCommandArgs); + // The args is not belongs to seatunnel, add into flink params + clientCommandArgs.setSeatunnelParams(jCommander.getUnknownOptions()); + if (clientCommandArgs.isHelp()) { + printHelp(jCommander); + } + return clientCommandArgs; + } + + public static ServerCommandArgs parseSeaTunnelServerArgs(String[] args) { + ServerCommandArgs serverCommandArgs = new ServerCommandArgs(); + JCommander jCommander = getJCommander(SERVER_SHELL_NAME, args, serverCommandArgs); + // The args is not belongs to seatunnel, add into flink params + serverCommandArgs.setSeatunnelParams(jCommander.getUnknownOptions()); + if (serverCommandArgs.isHelp()) { + printHelp(jCommander); + } + return serverCommandArgs; + } + + private static void printHelp(JCommander jCommander) { + jCommander.setUsageFormatter(new UnixStyleUsageFormatter(jCommander)); + jCommander.usage(); + System.exit(SeaTunnelConstant.USAGE_EXIT_CODE); + } + + private static JCommander getJCommander(String shellName, String[] args, Object serverCommandArgs) { + return JCommander.newBuilder() + .programName(shellName) + .addObject(serverCommandArgs) .acceptUnknownOptions(true) .args(args) .build(); - // The args is not belongs to seatunnel, add into flink params - seatunnelCommandArgs.setSeatunnelParams(jCommander.getUnknownOptions()); - if (seatunnelCommandArgs.isHelp()) { - jCommander.setUsageFormatter(new UnixStyleUsageFormatter(jCommander)); - jCommander.usage(); - System.exit(SeaTunnelConstant.USAGE_EXIT_CODE); - } - return seatunnelCommandArgs; } + } diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClient.java similarity index 73% rename from seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java rename to seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClient.java index a3a98a7b184..0f092781b37 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnel.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelClient.java @@ -20,14 +20,14 @@ import org.apache.seatunnel.core.starter.Seatunnel; import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.exception.CommandException; -import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; -import org.apache.seatunnel.core.starter.seatunnel.command.SeaTunnelCommandBuilder; +import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.command.ClientCommandBuilder; -public class SeaTunnel { +public class SeaTunnelClient { public static void main(String[] args) throws CommandException { - SeaTunnelCommandArgs seaTunnelCommandArgs = CommandLineUtils.parseSeaTunnelArgs(args); - Command command = - new SeaTunnelCommandBuilder().buildCommand(seaTunnelCommandArgs); + ClientCommandArgs clientCommandArgs = CommandLineUtils.parseSeaTunnelClientArgs(args); + Command command = + new ClientCommandBuilder().buildCommand(clientCommandArgs); Seatunnel.run(command); } } diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelServer.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelServer.java new file mode 100644 index 00000000000..a15adeb9fb9 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/SeaTunnelServer.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel; + +import org.apache.seatunnel.core.starter.Seatunnel; +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.exception.CommandException; +import org.apache.seatunnel.core.starter.seatunnel.args.ServerCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.command.ServerCommandBuilder; + +public class SeaTunnelServer { + public static void main(String[] args) throws CommandException { + ServerCommandArgs serverCommandArgs = CommandLineUtils.parseSeaTunnelServerArgs(args); + Command command = + new ServerCommandBuilder().buildCommand(serverCommandArgs); + Seatunnel.run(command); + } +} diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgs.java similarity index 86% rename from seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java rename to seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgs.java index 9e86b280005..371fb7c7682 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/SeaTunnelCommandArgs.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgs.java @@ -17,16 +17,14 @@ package org.apache.seatunnel.core.starter.seatunnel.args; -import org.apache.seatunnel.common.config.DeployMode; import org.apache.seatunnel.core.starter.command.AbstractCommandArgs; -import org.apache.seatunnel.core.starter.config.EngineType; import org.apache.seatunnel.engine.common.runtime.ExecutionMode; import com.beust.jcommander.Parameter; import java.util.List; -public class SeaTunnelCommandArgs extends AbstractCommandArgs { +public class ClientCommandArgs extends AbstractCommandArgs { /** * Undefined parameters parsed will be stored here as seatunnel engine command parameters. @@ -37,54 +35,45 @@ public class SeaTunnelCommandArgs extends AbstractCommandArgs { description = "The name of job") private String name = "seatunnel_job"; - @Parameter(names = {"-cn", "--cluster"}, - description = "The name of cluster") - private String clusterName = "seatunnel_default_cluster"; - @Parameter(names = {"-e", "--deploy-mode"}, description = "SeaTunnel deploy mode", converter = ExecutionModeConverter.class) private ExecutionMode executionMode = ExecutionMode.LOCAL; - public ExecutionMode getExecutionMode() { - return executionMode; - } + @Parameter(names = {"-cn", "--cluster"}, + description = "The name of cluster") + private String clusterName = "seatunnel_default_cluster"; - public void setExecutionMode(ExecutionMode executionMode) { - this.executionMode = executionMode; + public String getClusterName() { + return clusterName; } - public String getName() { - return name; + public void setClusterName(String clusterName) { + this.clusterName = clusterName; } - public void setName(String name) { - this.name = name; + public List getSeatunnelParams() { + return seatunnelParams; } - public String getClusterName() { - return clusterName; + public void setSeatunnelParams(List seatunnelParams) { + this.seatunnelParams = seatunnelParams; } - public void setClusterName(String clusterName) { - this.clusterName = clusterName; + public ExecutionMode getExecutionMode() { + return executionMode; } - @Override - public EngineType getEngineType() { - return EngineType.SEATUNNEL; + public void setExecutionMode(ExecutionMode executionMode) { + this.executionMode = executionMode; } - @Override - public DeployMode getDeployMode() { - return DeployMode.CLIENT; + public String getName() { + return name; } - public List getSeatunnelParams() { - return seatunnelParams; + public void setName(String name) { + this.name = name; } - public void setSeatunnelParams(List seatunnelParams) { - this.seatunnelParams = seatunnelParams; - } } diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ServerCommandArgs.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ServerCommandArgs.java new file mode 100644 index 00000000000..ec0019eb944 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ServerCommandArgs.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.args; + +import org.apache.seatunnel.core.starter.command.CommandArgs; + +import com.beust.jcommander.Parameter; + +import java.util.List; + +public class ServerCommandArgs implements CommandArgs { + + /** + * Undefined parameters parsed will be stored here as seatunnel engine command parameters. + */ + private List seatunnelParams; + + @Parameter(names = {"-cn", "--cluster"}, + description = "The name of cluster") + private String clusterName = "seatunnel_default_cluster"; + + @Parameter(names = {"-h", "--help"}, + help = true, + description = "Show the usage message") + private boolean help = false; + + public boolean isHelp() { + return help; + } + + public void setHelp(boolean help) { + this.help = help; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public List getSeatunnelParams() { + return seatunnelParams; + } + + public void setSeatunnelParams(List seatunnelParams) { + this.seatunnelParams = seatunnelParams; + } +} diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ApiConfValidateCommand.java similarity index 77% rename from seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java rename to seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ApiConfValidateCommand.java index 8036120ae44..20bc93659f5 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiConfValidateCommand.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ApiConfValidateCommand.java @@ -20,7 +20,7 @@ import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.config.ConfigBuilder; import org.apache.seatunnel.core.starter.exception.ConfigCheckException; -import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; import org.apache.seatunnel.core.starter.seatunnel.config.SeaTunnelApiConfigChecker; import org.apache.seatunnel.core.starter.utils.FileUtils; @@ -32,19 +32,19 @@ /** * Use to validate the configuration of the SeaTunnel API. */ -public class SeaTunnelApiConfValidateCommand implements Command { +public class ApiConfValidateCommand implements Command { - private static final Logger LOGGER = LoggerFactory.getLogger(SeaTunnelApiConfValidateCommand.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ApiConfValidateCommand.class); - private final SeaTunnelCommandArgs seaTunnelCommandArgs; + private final ClientCommandArgs clientCommandArgs; - public SeaTunnelApiConfValidateCommand(SeaTunnelCommandArgs seaTunnelCommandArgs) { - this.seaTunnelCommandArgs = seaTunnelCommandArgs; + public ApiConfValidateCommand(ClientCommandArgs clientCommandArgs) { + this.clientCommandArgs = clientCommandArgs; } @Override public void execute() throws ConfigCheckException { - Path configPath = FileUtils.getConfigPath(seaTunnelCommandArgs); + Path configPath = FileUtils.getConfigPath(clientCommandArgs); ConfigBuilder configBuilder = new ConfigBuilder(configPath); new SeaTunnelApiConfigChecker().checkConfig(configBuilder.getConfig()); LOGGER.info("config OK !"); diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientCommandBuilder.java similarity index 73% rename from seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java rename to seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientCommandBuilder.java index 722c84e7b0d..fb9bfc6d04c 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelCommandBuilder.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientCommandBuilder.java @@ -20,14 +20,14 @@ import org.apache.seatunnel.common.config.Common; import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.command.CommandBuilder; -import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; -public class SeaTunnelCommandBuilder implements CommandBuilder { +public class ClientCommandBuilder implements CommandBuilder { @Override - public Command buildCommand(SeaTunnelCommandArgs commandArgs) { + public Command buildCommand(ClientCommandArgs commandArgs) { Common.setDeployMode(commandArgs.getDeployMode()); - return commandArgs.isCheckConfig() ? new SeaTunnelApiConfValidateCommand(commandArgs) - : new SeaTunnelApiTaskExecuteCommand(commandArgs); + return commandArgs.isCheckConfig() ? new ApiConfValidateCommand(commandArgs) + : new ClientExecuteCommand(commandArgs); } } diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java similarity index 83% rename from seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java rename to seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java index d9d8638d9a7..f7ca752ad44 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/SeaTunnelApiTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java @@ -19,7 +19,7 @@ import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.exception.CommandExecuteException; -import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; import org.apache.seatunnel.core.starter.utils.FileUtils; import org.apache.seatunnel.engine.client.SeaTunnelClient; import org.apache.seatunnel.engine.client.job.ClientJobProxy; @@ -41,26 +41,24 @@ /** * This command is used to execute the SeaTunnel engine job by SeaTunnel API. */ -public class SeaTunnelApiTaskExecuteCommand implements Command { +public class ClientExecuteCommand implements Command { - private final SeaTunnelCommandArgs seaTunnelCommandArgs; + private final ClientCommandArgs clientCommandArgs; - // TODO custom cluster name on cluster execution mode - - public SeaTunnelApiTaskExecuteCommand(SeaTunnelCommandArgs seaTunnelCommandArgs) { - this.seaTunnelCommandArgs = seaTunnelCommandArgs; + public ClientExecuteCommand(ClientCommandArgs clientCommandArgs) { + this.clientCommandArgs = clientCommandArgs; } @Override public void execute() throws CommandExecuteException { - Path configFile = FileUtils.getConfigPath(seaTunnelCommandArgs); + Path configFile = FileUtils.getConfigPath(clientCommandArgs); JobConfig jobConfig = new JobConfig(); - jobConfig.setName(seaTunnelCommandArgs.getName()); + jobConfig.setName(clientCommandArgs.getName()); HazelcastInstance instance = null; try { - String clusterName = seaTunnelCommandArgs.getClusterName(); - if (seaTunnelCommandArgs.getExecutionMode().equals(ExecutionMode.LOCAL)) { + String clusterName = clientCommandArgs.getClusterName(); + if (clientCommandArgs.getExecutionMode().equals(ExecutionMode.LOCAL)) { clusterName = creatRandomClusterName(clusterName); instance = createServerInLocal(clusterName); } diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerCommandBuilder.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerCommandBuilder.java new file mode 100644 index 00000000000..999bd7893fc --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerCommandBuilder.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.command; + +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.command.CommandBuilder; +import org.apache.seatunnel.core.starter.seatunnel.args.ServerCommandArgs; + +public class ServerCommandBuilder implements CommandBuilder { + + @Override + public Command buildCommand(ServerCommandArgs commandArgs) { + return new ServerExecuteCommand(commandArgs); + } +} diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommand.java new file mode 100644 index 00000000000..bbd50fd56cd --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ServerExecuteCommand.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.command; + +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.seatunnel.args.ServerCommandArgs; +import org.apache.seatunnel.engine.common.config.ConfigProvider; +import org.apache.seatunnel.engine.common.config.SeaTunnelConfig; +import org.apache.seatunnel.engine.server.SeaTunnelNodeContext; + +import com.hazelcast.instance.impl.HazelcastInstanceFactory; + +/** + * This command is used to execute the SeaTunnel engine job by SeaTunnel API. + */ +public class ServerExecuteCommand implements Command { + + private final ServerCommandArgs serverCommandArgs; + + public ServerExecuteCommand(ServerCommandArgs serverCommandArgs) { + this.serverCommandArgs = serverCommandArgs; + } + + @Override + public void execute() { + SeaTunnelConfig seaTunnelConfig = ConfigProvider.locateAndGetSeaTunnelConfig(); + seaTunnelConfig.getHazelcastConfig().setClusterName(serverCommandArgs.getClusterName()); + HazelcastInstanceFactory.newHazelcastInstance(seaTunnelConfig.getHazelcastConfig(), + Thread.currentThread().getName(), + new SeaTunnelNodeContext(seaTunnelConfig)); + } + +} diff --git a/seatunnel-engine/seatunnel-engine-client/pom.xml b/seatunnel-engine/seatunnel-engine-client/pom.xml index b24ad2088a0..c8e6c819cc5 100644 --- a/seatunnel-engine/seatunnel-engine-client/pom.xml +++ b/seatunnel-engine/seatunnel-engine-client/pom.xml @@ -41,7 +41,7 @@ org.apache.seatunnel - seatunnel-core-base + seatunnel-core-starter ${project.version} @@ -68,7 +68,7 @@ org.apache.seatunnel - connector-file-local + connector-console ${project.version} test diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/resources/client_test.conf b/seatunnel-engine/seatunnel-engine-client/src/test/resources/client_test.conf index 69ce5614a12..65875938eb8 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/resources/client_test.conf +++ b/seatunnel-engine/seatunnel-engine-client/src/test/resources/client_test.conf @@ -45,19 +45,7 @@ transform { } sink { - LocalFile { - path="file:///tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="text" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error", + console { source_table_name="fake" } } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-engine-examples/pom.xml b/seatunnel-examples/seatunnel-engine-examples/pom.xml index b75a68b856c..54150c58452 100644 --- a/seatunnel-examples/seatunnel-engine-examples/pom.xml +++ b/seatunnel-examples/seatunnel-engine-examples/pom.xml @@ -30,7 +30,7 @@ org.apache.seatunnel - seatunnel-seatunnel-starter + seatunnel-starter ${project.version} diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java index a50f312712b..dd4e5fe5a1b 100644 --- a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java @@ -20,8 +20,8 @@ import org.apache.seatunnel.core.starter.Seatunnel; import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.exception.CommandException; -import org.apache.seatunnel.core.starter.seatunnel.args.SeaTunnelCommandArgs; -import org.apache.seatunnel.core.starter.seatunnel.command.SeaTunnelCommandBuilder; +import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.command.ClientCommandBuilder; import java.io.FileNotFoundException; import java.net.URISyntaxException; @@ -32,12 +32,12 @@ public class SeaTunnelEngineExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException, CommandException { String configFile = getTestConfigFile("/examples/fake_to_console.conf"); - SeaTunnelCommandArgs seaTunnelCommandArgs = new SeaTunnelCommandArgs(); - seaTunnelCommandArgs.setConfigFile(configFile); - seaTunnelCommandArgs.setCheckConfig(false); - seaTunnelCommandArgs.setName("fake_to_console"); - Command command = - new SeaTunnelCommandBuilder().buildCommand(seaTunnelCommandArgs); + ClientCommandArgs clientCommandArgs = new ClientCommandArgs(); + clientCommandArgs.setConfigFile(configFile); + clientCommandArgs.setCheckConfig(false); + clientCommandArgs.setName("fake_to_console"); + Command command = + new ClientCommandBuilder().buildCommand(clientCommandArgs); Seatunnel.run(command); } From bcb7659da0f478a5f0f1c8c8985550a83281ca8a Mon Sep 17 00:00:00 2001 From: Hisoka Date: Thu, 8 Sep 2022 21:54:37 +0800 Subject: [PATCH 10/22] [Engine] [Starter] Add seatunnel engine e2e --- .../spark/config/SparkApiConfigChecker.java | 1 - .../src/main/bin/seatunnel-cluster.sh | 2 +- .../seatunnel/args/ClientCommandArgs.java | 13 +- seatunnel-e2e/seatunnel-engine-e2e/pom.xml | 4 + .../pom.xml | 16 ++- .../e2e/file/FakeSourceToConsoleIT.java} | 18 ++- .../test/resources/fakesource_to_console.conf | 42 ++++++ .../seatunnel-engine-e2e-base/pom.xml | 23 ++- .../e2e/{engine => }/JobExecutionIT.java | 3 +- .../engine/e2e/SeaTunnelContainer.java | 133 ++++++++++++++++++ .../engine/client/job/ClientJobProxy.java | 1 + .../engine/server/task/SeaTunnelTask.java | 7 +- .../engine/SeaTunnelEngineExample.java | 3 + .../engine/SeaTunnelEngineServerExample.java | 34 +++++ 14 files changed, 283 insertions(+), 17 deletions(-) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/SeaTunnelContainer.java => seatunnel-engine-connector-file-e2e/src/test/java/org/apache/seatunnel/engine/e2e/file/FakeSourceToConsoleIT.java} (59%) create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/resources/fakesource_to_console.conf rename seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/{engine => }/JobExecutionIT.java (98%) create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java create mode 100644 seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java index affa28cdac9..afdf775796a 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/config/SparkApiConfigChecker.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.core.starter.config.ConfigChecker; import org.apache.seatunnel.core.starter.exception.ConfigCheckException; -import org.apache.seatunnel.spark.SparkEnvironment; import org.apache.seatunnel.shade.com.typesafe.config.Config; diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh index debf019999e..228d07d0181 100755 --- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh +++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh @@ -45,7 +45,7 @@ fi if [ $# == 0 ] then - args="-h" + args="" else args=$@ fi diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgs.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgs.java index 371fb7c7682..d5202b23c4c 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgs.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ClientCommandArgs.java @@ -17,7 +17,9 @@ package org.apache.seatunnel.core.starter.seatunnel.args; +import org.apache.seatunnel.common.config.DeployMode; import org.apache.seatunnel.core.starter.command.AbstractCommandArgs; +import org.apache.seatunnel.core.starter.config.EngineType; import org.apache.seatunnel.engine.common.runtime.ExecutionMode; import com.beust.jcommander.Parameter; @@ -38,7 +40,7 @@ public class ClientCommandArgs extends AbstractCommandArgs { @Parameter(names = {"-e", "--deploy-mode"}, description = "SeaTunnel deploy mode", converter = ExecutionModeConverter.class) - private ExecutionMode executionMode = ExecutionMode.LOCAL; + private ExecutionMode executionMode = ExecutionMode.CLUSTER; @Parameter(names = {"-cn", "--cluster"}, description = "The name of cluster") @@ -76,4 +78,13 @@ public void setName(String name) { this.name = name; } + @Override + public EngineType getEngineType() { + return EngineType.SEATUNNEL; + } + + @Override + public DeployMode getDeployMode() { + return DeployMode.CLIENT; + } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/pom.xml index ea38a9b374f..0021f14077b 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/pom.xml @@ -72,6 +72,10 @@ seatunnel-engine-server ${project.version} + + org.testcontainers + testcontainers + org.awaitility awaitility diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml index 1b73ea3a07f..18b481f6eab 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml @@ -26,11 +26,17 @@ 4.0.0 - seatunnel-engine-connector-v2-e2e + seatunnel-engine-connector-file-e2e - - 8 - 8 - + + + org.apache.seatunnel + seatunnel-engine-e2e-base + ${project.version} + tests + test-jar + test + + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/java/org/apache/seatunnel/engine/e2e/file/FakeSourceToConsoleIT.java similarity index 59% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/SeaTunnelContainer.java rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/java/org/apache/seatunnel/engine/e2e/file/FakeSourceToConsoleIT.java index 0e8d200aa7a..66b20140cdf 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/SeaTunnelContainer.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/java/org/apache/seatunnel/engine/e2e/file/FakeSourceToConsoleIT.java @@ -15,7 +15,21 @@ * limitations under the License. */ -package org.apache.seatunnel.engine.e2e.engine; +package org.apache.seatunnel.engine.e2e.file; -public abstract class SeaTunnelContainer { +import org.apache.seatunnel.engine.e2e.SeaTunnelContainer; + +import org.junit.Assert; +import org.junit.Test; +import org.testcontainers.containers.Container; + +import java.io.IOException; + +public class FakeSourceToConsoleIT extends SeaTunnelContainer { + + @Test + public void testFakeSourceToConsoleSink() throws IOException, InterruptedException { + Container.ExecResult execResult = executeSeaTunnelJob("/fakesource_to_console.conf"); + Assert.assertEquals(0, execResult.getExitCode()); + } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/resources/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/resources/fakesource_to_console.conf new file mode 100644 index 00000000000..ae94a82c1d3 --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/resources/fakesource_to_console.conf @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### + +env { + job.mode = "BATCH" +} + +source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** + FakeSource { + result_table_name = "fake" + field_name = "name,age" + } +} + +transform { +} + +sink { + console { + } + + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml index 8ea876392cc..5dc8f3c6db7 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml @@ -29,8 +29,27 @@ seatunnel-engine-e2e-base - 8 - 8 + 2.4 + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + false + + + + + test-jar + + + + + + + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobExecutionIT.java similarity index 98% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobExecutionIT.java index 84a247bbfcb..b1c444aceb8 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/engine/JobExecutionIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobExecutionIT.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.seatunnel.engine.e2e.engine; +package org.apache.seatunnel.engine.e2e; import org.apache.seatunnel.common.config.Common; import org.apache.seatunnel.common.config.DeployMode; @@ -27,7 +27,6 @@ import org.apache.seatunnel.engine.common.config.SeaTunnelClientConfig; import org.apache.seatunnel.engine.common.config.SeaTunnelConfig; import org.apache.seatunnel.engine.core.job.JobStatus; -import org.apache.seatunnel.engine.e2e.TestUtils; import org.apache.seatunnel.engine.server.SeaTunnelNodeContext; import com.google.common.collect.Lists; diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java new file mode 100644 index 00000000000..9b6495850da --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.e2e; + +import org.junit.After; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +public abstract class SeaTunnelContainer { + + private static final Logger LOG = LoggerFactory.getLogger(SeaTunnelContainer.class); + private static final String JDK_DOCKER_IMAGE = "openjdk:8"; + protected static final Network NETWORK = Network.newNetwork(); + private static final Path PROJECT_ROOT_PATH = Paths.get(System.getProperty("user.dir")).getParent().getParent().getParent(); + + private static final String SEATUNNEL_HOME = "/tmp/seatunnel"; + private static final String PLUGIN_MAPPING_FILE = "plugin-mapping.properties"; + private static final String SEATUNNEL_BIN = Paths.get(SEATUNNEL_HOME, "bin").toString(); + private static final String SEATUNNEL_LIB = Paths.get(SEATUNNEL_HOME, "lib").toString(); + private static final String SEATUNNEL_CONNECTORS = Paths.get(SEATUNNEL_HOME, "connectors").toString(); + private static final String CLIENT_SHELL = "seatunnel.sh"; + private static final String SERVER_SHELL = "seatunnel-cluster.sh"; + private GenericContainer server; + + @Before + public void before() { + Map mountMapping = getFileMountMapping(); + server = new GenericContainer<>(JDK_DOCKER_IMAGE) + .withNetwork(NETWORK) + .withCommand(Paths.get(SEATUNNEL_BIN, SERVER_SHELL).toString()) + .withNetworkAliases("server") + .withExposedPorts() + .withLogConsumer(new Slf4jLogConsumer(LOG)) + .waitingFor(Wait.forLogMessage(".*received new worker register.*\\n", 1)); + mountMapping.forEach(server::withFileSystemBind); + server.start(); + } + + protected Map getFileMountMapping() { + + Map mountMapping = new HashMap<>(); + // copy lib + mountMapping.put(PROJECT_ROOT_PATH + "/seatunnel-core/seatunnel-starter/target/seatunnel-starter.jar", + Paths.get(SEATUNNEL_LIB, "seatunnel-starter.jar").toString()); + + // copy bin + mountMapping.put(PROJECT_ROOT_PATH + "/seatunnel-core/seatunnel-starter/src/main/bin/", + Paths.get(SEATUNNEL_BIN).toString()); + + // copy connectors + mountMapping.put(PROJECT_ROOT_PATH + + "/seatunnel-connectors-v2-dist/target/lib", Paths.get(SEATUNNEL_CONNECTORS, "seatunnel").toString()); + + // copy plugin-mapping.properties + mountMapping.put(PROJECT_ROOT_PATH + "/plugin-mapping.properties", Paths.get(SEATUNNEL_CONNECTORS, PLUGIN_MAPPING_FILE).toString()); + + return mountMapping; + } + + @SuppressWarnings("checkstyle:MagicNumber") + public Container.ExecResult executeSeaTunnelJob(String confFile) throws IOException, InterruptedException { + final String confPath = getResource(confFile); + if (!new File(confPath).exists()) { + throw new IllegalArgumentException(confFile + " doesn't exist"); + } + final String targetConfInContainer = Paths.get("/tmp", confFile).toString(); + server.copyFileToContainer(MountableFile.forHostPath(confPath), targetConfInContainer); + + // Running IT use cases under Windows requires replacing \ with / + String conf = targetConfInContainer.replaceAll("\\\\", "/"); + final List command = new ArrayList<>(); + command.add(Paths.get(SEATUNNEL_BIN, CLIENT_SHELL).toString()); + command.add("--config " + conf); + + CompletableFuture.runAsync(() -> { + try { + Thread.sleep(15000); + // cancel server if bash command not return + server.stop(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + Container.ExecResult execResult = server.execInContainer("bash", "-c", String.join(" ", command)); + LOG.info(execResult.getStdout()); + LOG.error(execResult.getStderr()); + return execResult; + } + + @After + public void after() { + if (server != null) { + server.close(); + } + } + + private String getResource(String confFile) { + return System.getProperty("user.dir") + "/src/test/resources" + confFile; + } + +} diff --git a/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java b/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java index 8c372f2d568..aa18dd7e52f 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java +++ b/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java @@ -53,6 +53,7 @@ public long getJobId() { } private void submitJob() { + LOGGER.info(String.format("start submit job, job id: %s, with plugin jar %s", jobImmutableInformation.getJobId(), jobImmutableInformation.getPluginJarsUrls())); ClientMessage request = SeaTunnelSubmitJobCodec.encodeRequest(jobImmutableInformation.getJobId(), seaTunnelHazelcastClient.getSerializationService().toData(jobImmutableInformation)); PassiveCompletableFuture submitJobFuture = diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/SeaTunnelTask.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/SeaTunnelTask.java index cd10dc5292d..f253a3fa9e9 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/SeaTunnelTask.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/SeaTunnelTask.java @@ -49,7 +49,6 @@ import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -161,7 +160,8 @@ protected void checkDone() { @Override public Set getJarsUrl() { - List now = Collections.singletonList(executionFlow); + List now = new ArrayList<>(); + now.add(executionFlow); Set urls = new HashSet<>(); List next = new ArrayList<>(); while (!now.isEmpty()) { @@ -172,7 +172,8 @@ public Set getJarsUrl() { } next.addAll(n.getNext()); }); - now = next; + now.clear(); + now.addAll(next); } return urls; } diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java index dd4e5fe5a1b..f5f04e43080 100644 --- a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineExample.java @@ -22,6 +22,7 @@ import org.apache.seatunnel.core.starter.exception.CommandException; import org.apache.seatunnel.core.starter.seatunnel.args.ClientCommandArgs; import org.apache.seatunnel.core.starter.seatunnel.command.ClientCommandBuilder; +import org.apache.seatunnel.engine.common.runtime.ExecutionMode; import java.io.FileNotFoundException; import java.net.URISyntaxException; @@ -36,6 +37,8 @@ public static void main(String[] args) throws FileNotFoundException, URISyntaxEx clientCommandArgs.setConfigFile(configFile); clientCommandArgs.setCheckConfig(false); clientCommandArgs.setName("fake_to_console"); + // Change Execution Mode to CLUSTER to use client mode, before do this, you should start SeaTunnelEngineServerExample + clientCommandArgs.setExecutionMode(ExecutionMode.CLUSTER); Command command = new ClientCommandBuilder().buildCommand(clientCommandArgs); Seatunnel.run(command); diff --git a/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java new file mode 100644 index 00000000000..af28f2872f9 --- /dev/null +++ b/seatunnel-examples/seatunnel-engine-examples/src/main/java/org/apache/seatunnel/example/engine/SeaTunnelEngineServerExample.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.example.engine; + +import org.apache.seatunnel.core.starter.Seatunnel; +import org.apache.seatunnel.core.starter.command.Command; +import org.apache.seatunnel.core.starter.exception.CommandException; +import org.apache.seatunnel.core.starter.seatunnel.args.ServerCommandArgs; +import org.apache.seatunnel.core.starter.seatunnel.command.ServerCommandBuilder; + +public class SeaTunnelEngineServerExample { + public static void main(String[] args) throws CommandException { + ServerCommandArgs serverCommandArgs = new ServerCommandArgs(); + Command command = + new ServerCommandBuilder().buildCommand(serverCommandArgs); + Seatunnel.run(command); + } + +} From 882f0a5be64b800acca35f1ccaf43f7b7e036c8a Mon Sep 17 00:00:00 2001 From: Hisoka Date: Thu, 8 Sep 2022 22:03:53 +0800 Subject: [PATCH 11/22] [Engine] [Starter] Add seatunnel engine e2e --- .../args/ExecutionModeConverter.java | 34 +++++++++++++++++++ .../engine/server/task/SeaTunnelTask.java | 1 + 2 files changed, 35 insertions(+) create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java new file mode 100644 index 00000000000..ee7da538b9f --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/args/ExecutionModeConverter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.args; + +import org.apache.seatunnel.engine.common.runtime.ExecutionMode; + +import com.beust.jcommander.IStringConverter; +import com.beust.jcommander.ParameterException; + +public class ExecutionModeConverter implements IStringConverter { + @Override + public ExecutionMode convert(String value) { + try { + return ExecutionMode.valueOf(value.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new ParameterException("execution-mode: " + value + " is not allowed."); + } + } +} diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/SeaTunnelTask.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/SeaTunnelTask.java index 4cb23fde04e..003989c7ba8 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/SeaTunnelTask.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/SeaTunnelTask.java @@ -68,6 +68,7 @@ import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; From 51e176b1ec008ba3a3366255f0e4be3af6620fec Mon Sep 17 00:00:00 2001 From: Hisoka Date: Thu, 8 Sep 2022 22:08:53 +0800 Subject: [PATCH 12/22] [Engine] [Starter] Add seatunnel engine e2e --- seatunnel-e2e/seatunnel-engine-e2e/pom.xml | 2 +- .../pom.xml | 2 +- .../seatunnel/engine/e2e/console}/FakeSourceToConsoleIT.java | 2 +- .../src/test/resources/fakesource_to_console.conf | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-connector-file-e2e => seatunnel-engine-connector-console-e2e}/pom.xml (96%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-connector-file-e2e/src/test/java/org/apache/seatunnel/engine/e2e/file => seatunnel-engine-connector-console-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console}/FakeSourceToConsoleIT.java (96%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-connector-file-e2e => seatunnel-engine-connector-console-e2e}/src/test/resources/fakesource_to_console.conf (100%) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/pom.xml index 0021f14077b..4cfcda978d6 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/pom.xml @@ -31,7 +31,7 @@ seatunnel-engine-e2e-base - seatunnel-engine-connector-file-e2e + seatunnel-engine-connector-console-e2e diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/pom.xml similarity index 96% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/pom.xml index 18b481f6eab..b7ea442aeb5 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - seatunnel-engine-connector-file-e2e + seatunnel-engine-connector-console-e2e diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/java/org/apache/seatunnel/engine/e2e/file/FakeSourceToConsoleIT.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java similarity index 96% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/java/org/apache/seatunnel/engine/e2e/file/FakeSourceToConsoleIT.java rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java index 66b20140cdf..fa63feb1c1c 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/java/org/apache/seatunnel/engine/e2e/file/FakeSourceToConsoleIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.seatunnel.engine.e2e.file; +package org.apache.seatunnel.engine.e2e.console; import org.apache.seatunnel.engine.e2e.SeaTunnelContainer; diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/resources/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/resources/fakesource_to_console.conf similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-file-e2e/src/test/resources/fakesource_to_console.conf rename to seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/resources/fakesource_to_console.conf From b13054296cc145d7781eadbf4102adae5d36b645 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Fri, 9 Sep 2022 10:04:27 +0800 Subject: [PATCH 13/22] [Engine] [Starter] add miss class file --- .../config/SeaTunnelApiConfigChecker.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java new file mode 100644 index 00000000000..bfaff2a9c35 --- /dev/null +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/config/SeaTunnelApiConfigChecker.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.core.starter.seatunnel.config; + +import org.apache.seatunnel.core.starter.config.ConfigChecker; +import org.apache.seatunnel.core.starter.exception.ConfigCheckException; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +public class SeaTunnelApiConfigChecker implements ConfigChecker { + + @Override + public void checkConfig(Config config) throws ConfigCheckException { + // TODO implement + } +} From a326c5b623dc5f88520de3ccf8b6b9b84e5f1e18 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Fri, 9 Sep 2022 10:47:42 +0800 Subject: [PATCH 14/22] [Engine] [Starter] fix ci problems --- seatunnel-engine/seatunnel-engine-client/pom.xml | 7 +++++++ .../src/test/resources/client_test.conf | 16 ++++++++++++++-- .../server/task/flow/SinkFlowLifeCycle.java | 12 ++++++++---- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/seatunnel-engine/seatunnel-engine-client/pom.xml b/seatunnel-engine/seatunnel-engine-client/pom.xml index c8e6c819cc5..f8e51c2fc11 100644 --- a/seatunnel-engine/seatunnel-engine-client/pom.xml +++ b/seatunnel-engine/seatunnel-engine-client/pom.xml @@ -72,6 +72,13 @@ ${project.version} test + + org.apache.seatunnel + connector-file-local + ${project.version} + test + + com.hazelcast diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/resources/client_test.conf b/seatunnel-engine/seatunnel-engine-client/src/test/resources/client_test.conf index 65875938eb8..e2634dfa8b7 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/resources/client_test.conf +++ b/seatunnel-engine/seatunnel-engine-client/src/test/resources/client_test.conf @@ -45,7 +45,19 @@ transform { } sink { - console { - source_table_name="fake" + LocalFile { + path = "file:///tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["age"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "text" + sink_columns = ["name", "age"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error", + source_table_name = "fake" } } \ No newline at end of file diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/SinkFlowLifeCycle.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/SinkFlowLifeCycle.java index a3aa29fc2a0..f3538665f08 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/SinkFlowLifeCycle.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/flow/SinkFlowLifeCycle.java @@ -116,17 +116,21 @@ public void received(Record record) { prepareClose = true; } if (barrier.snapshot()) { - if (writerStateSerializer.isPresent()) { + if (!writerStateSerializer.isPresent()) { runningTask.addState(barrier, sinkAction.getId(), Collections.emptyList()); } else { List states = writer.snapshotState(barrier.getId()); runningTask.addState(barrier, sinkAction.getId(), serializeStates(writerStateSerializer.get(), states)); } // TODO: prepare commit - runningTask.getExecutionContext().sendToMember(new SinkPrepareCommitOperation(barrier, committerTaskLocation, - new byte[0]), committerTaskAddress); + if (containCommitter) { + runningTask.getExecutionContext().sendToMember(new SinkPrepareCommitOperation(barrier, committerTaskLocation, + new byte[0]), committerTaskAddress); + } } else { - runningTask.getExecutionContext().sendToMember(new CheckpointBarrierTriggerOperation(barrier, committerTaskLocation), committerTaskAddress); + if (containCommitter) { + runningTask.getExecutionContext().sendToMember(new CheckpointBarrierTriggerOperation(barrier, committerTaskLocation), committerTaskAddress); + } } runningTask.ack(barrier); } else { From 00bc32e062523798ef6690b8e99fd271ac7f0453 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Fri, 9 Sep 2022 12:53:45 +0800 Subject: [PATCH 15/22] [Engine] [Starter] rename e2e module --- .../seatunnel/command/ClientExecuteCommand.java | 5 ++++- .../pom.xml | 4 ++-- .../engine/e2e/console/FakeSourceToConsoleIT.java | 0 .../src/test/resources/fakesource_to_console.conf | 2 -- .../pom.xml | 2 +- .../apache/seatunnel/engine/e2e/JobExecutionIT.java | 0 .../seatunnel/engine/e2e/SeaTunnelContainer.java | 0 .../org/apache/seatunnel/engine/e2e/TestUtils.java | 0 .../src/test/resources/batch_fakesource_to_file.conf | 0 .../resources/batch_fakesource_to_file_complex.conf | 0 .../streaming_fakesource_to_file_complex.conf | 0 seatunnel-e2e/seatunnel-engine-e2e/pom.xml | 4 ++-- .../seatunnel/engine/client/job/ClientJobProxy.java | 6 +++++- .../java/org/apache/seatunnel/engine/core/job/Job.java | 2 ++ seatunnel-examples/seatunnel-engine-examples/pom.xml | 10 ---------- 15 files changed, 16 insertions(+), 19 deletions(-) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-connector-console-e2e => connector-console-seatunnel-e2e}/pom.xml (92%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-connector-console-e2e => connector-console-seatunnel-e2e}/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-connector-console-e2e => connector-console-seatunnel-e2e}/src/test/resources/fakesource_to_console.conf (84%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-e2e-base => connector-seatunnel-e2e-base}/pom.xml (97%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-e2e-base => connector-seatunnel-e2e-base}/src/test/java/org/apache/seatunnel/engine/e2e/JobExecutionIT.java (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-e2e-base => connector-seatunnel-e2e-base}/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-e2e-base => connector-seatunnel-e2e-base}/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-e2e-base => connector-seatunnel-e2e-base}/src/test/resources/batch_fakesource_to_file.conf (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-e2e-base => connector-seatunnel-e2e-base}/src/test/resources/batch_fakesource_to_file_complex.conf (100%) rename seatunnel-e2e/seatunnel-engine-e2e/{seatunnel-engine-e2e-base => connector-seatunnel-e2e-base}/src/test/resources/streaming_fakesource_to_file_complex.conf (100%) diff --git a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java index f7ca752ad44..d99e708e906 100644 --- a/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java +++ b/seatunnel-core/seatunnel-starter/src/main/java/org/apache/seatunnel/core/starter/seatunnel/command/ClientExecuteCommand.java @@ -56,6 +56,7 @@ public void execute() throws CommandExecuteException { JobConfig jobConfig = new JobConfig(); jobConfig.setName(clientCommandArgs.getName()); HazelcastInstance instance = null; + ClientJobProxy clientJobProxy = null; try { String clusterName = clientCommandArgs.getClusterName(); if (clientCommandArgs.getExecutionMode().equals(ExecutionMode.LOCAL)) { @@ -67,7 +68,6 @@ public void execute() throws CommandExecuteException { SeaTunnelClient engineClient = new SeaTunnelClient(clientConfig); JobExecutionEnvironment jobExecutionEnv = engineClient.createExecutionContext(configFile.toString(), jobConfig); - ClientJobProxy clientJobProxy; clientJobProxy = jobExecutionEnv.execute(); clientJobProxy.waitForJobComplete(); } catch (ExecutionException | InterruptedException e) { @@ -76,6 +76,9 @@ public void execute() throws CommandExecuteException { if (instance != null) { instance.shutdown(); } + if (clientJobProxy != null) { + clientJobProxy.close(); + } } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/pom.xml similarity index 92% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/pom.xml rename to seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/pom.xml index b7ea442aeb5..afad8f533f7 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/pom.xml @@ -26,12 +26,12 @@ 4.0.0 - seatunnel-engine-connector-console-e2e + connector-console-seatunnel-e2e org.apache.seatunnel - seatunnel-engine-e2e-base + connector-seatunnel-e2e-base ${project.version} tests test-jar diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java rename to seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/java/org/apache/seatunnel/engine/e2e/console/FakeSourceToConsoleIT.java diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/resources/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/resources/fakesource_to_console.conf similarity index 84% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/resources/fakesource_to_console.conf rename to seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/resources/fakesource_to_console.conf index ae94a82c1d3..35f8633efac 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-connector-console-e2e/src/test/resources/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/resources/fakesource_to_console.conf @@ -37,6 +37,4 @@ sink { console { } - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml similarity index 97% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml rename to seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml index 5dc8f3c6db7..03f7c66bf2e 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - seatunnel-engine-e2e-base + connector-seatunnel-e2e-base 2.4 diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobExecutionIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobExecutionIT.java similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobExecutionIT.java rename to seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/JobExecutionIT.java diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java rename to seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java rename to seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/batch_fakesource_to_file.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file.conf similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/batch_fakesource_to_file.conf rename to seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file.conf diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf rename to seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf similarity index 100% rename from seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf rename to seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf diff --git a/seatunnel-e2e/seatunnel-engine-e2e/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/pom.xml index 4cfcda978d6..570bbefa7ad 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/pom.xml @@ -30,8 +30,8 @@ pom - seatunnel-engine-e2e-base - seatunnel-engine-connector-console-e2e + connector-seatunnel-e2e-base + connector-console-seatunnel-e2e diff --git a/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java b/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java index aa18dd7e52f..0441431080e 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java +++ b/seatunnel-engine/seatunnel-engine-client/src/main/java/org/apache/seatunnel/engine/client/job/ClientJobProxy.java @@ -85,10 +85,14 @@ public JobStatus waitForJobComplete() { jobImmutableInformation.getJobConfig().getName(), jobImmutableInformation.getJobId(), jobStatus)); - this.seaTunnelHazelcastClient.getHazelcastInstance().shutdown(); return jobStatus; } + @Override + public void close() { + this.seaTunnelHazelcastClient.getHazelcastInstance().shutdown(); + } + @Override public PassiveCompletableFuture doWaitForJobComplete() { return seaTunnelHazelcastClient.requestOnMasterAndGetCompletableFuture( diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/Job.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/Job.java index 517d5b1f28d..69ed8ad815d 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/Job.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/job/Job.java @@ -32,4 +32,6 @@ public interface Job { JobStatus getJobStatus(); JobStatus waitForJobComplete(); + + void close(); } diff --git a/seatunnel-examples/seatunnel-engine-examples/pom.xml b/seatunnel-examples/seatunnel-engine-examples/pom.xml index 54150c58452..a0d952b2c17 100644 --- a/seatunnel-examples/seatunnel-engine-examples/pom.xml +++ b/seatunnel-examples/seatunnel-engine-examples/pom.xml @@ -34,15 +34,5 @@ ${project.version} - - org.apache.seatunnel - connector-fake - ${project.version} - - - org.apache.seatunnel - connector-console - ${project.version} - \ No newline at end of file From 1f34c743e0be70ee0790c8006774bb26b122632c Mon Sep 17 00:00:00 2001 From: Hisoka Date: Fri, 9 Sep 2022 12:56:54 +0800 Subject: [PATCH 16/22] [Engine] [Starter] fix license --- seatunnel-dist/release-docs/LICENSE | 4 ---- tools/dependencies/known-dependencies.txt | 5 ----- 2 files changed, 9 deletions(-) diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE index d693eea8cb3..8fd55eb617e 100644 --- a/seatunnel-dist/release-docs/LICENSE +++ b/seatunnel-dist/release-docs/LICENSE @@ -345,9 +345,7 @@ The text of each license is the standard Apache 2.0 license. (Apache License, Version 2.0) Apache Commons CLI (commons-cli:commons-cli:1.4 - http://commons.apache.org/proper/commons-cli/) (Apache License, Version 2.0) Apache Commons Codec (commons-codec:commons-codec:1.13 - https://commons.apache.org/proper/commons-codec/) (Apache License, Version 2.0) Apache Commons Collections (org.apache.commons:commons-collections4:4.4 - https://commons.apache.org/proper/commons-collections/) - (Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.18 - https://commons.apache.org/proper/commons-compress/) (Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.20 - https://commons.apache.org/proper/commons-compress/) - (Apache License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.21 - https://commons.apache.org/proper/commons-compress/) (Apache License, Version 2.0) Apache Commons Configuration (org.apache.commons:commons-configuration2:2.1.1 - http://commons.apache.org/proper/commons-configuration/) (Apache License, Version 2.0) Apache Commons Crypto (org.apache.commons:commons-crypto:1.0.0 - http://commons.apache.org/proper/commons-crypto/) (Apache License, Version 2.0) Apache Commons Email (org.apache.commons:commons-email:1.5 - http://commons.apache.org/proper/commons-email/) @@ -627,7 +625,6 @@ The text of each license is the standard Apache 2.0 license. (The Apache Software License, Version 2.0) Apache Avro IPC (org.apache.avro:avro-ipc:1.8.2 - http://avro.apache.org) (The Apache Software License, Version 2.0) Apache Avro Mapred API (org.apache.avro:avro-mapred:1.8.2 - http://avro.apache.org/avro-mapred) (The Apache Software License, Version 2.0) Apache Commons CSV (org.apache.commons:commons-csv:1.0 - http://commons.apache.org/proper/commons-csv/) - (The Apache Software License, Version 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.8.1 - http://commons.apache.org/proper/commons-compress/) (The Apache Software License, Version 2.0) Apache Commons DBCP (org.apache.commons:commons-dbcp2:2.0.1 - http://commons.apache.org/proper/commons-dbcp/) (The Apache Software License, Version 2.0) Apache Commons Lang (org.apache.commons:commons-lang3:3.3.2 - http://commons.apache.org/proper/commons-lang/) (The Apache Software License, Version 2.0) Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/) @@ -678,7 +675,6 @@ The text of each license is the standard Apache 2.0 license. (The Apache Software License, Version 2.0) ClassMate (com.fasterxml:classmate:1.1.0 - http://github.com/cowtowncoder/java-classmate) (The Apache Software License, Version 2.0) Commons BeanUtils Core (commons-beanutils:commons-beanutils-core:1.8.0 - http://commons.apache.org/beanutils/) (The Apache Software License, Version 2.0) Commons CLI (commons-cli:commons-cli:1.2 - http://commons.apache.org/cli/) - (The Apache Software License, Version 2.0) Commons Compress (org.apache.commons:commons-compress:1.4.1 - http://commons.apache.org/compress/) (The Apache Software License, Version 2.0) Commons Configuration (commons-configuration:commons-configuration:1.6 - http://commons.apache.org/${pom.artifactId.substring(8)}/) (The Apache Software License, Version 2.0) Commons Configuration (commons-configuration:commons-configuration:1.7 - http://commons.apache.org/configuration/) (The Apache Software License, Version 2.0) Commons DBCP (commons-dbcp:commons-dbcp:1.4 - http://commons.apache.org/dbcp/) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 077ddc9660d..f662660a98f 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -68,12 +68,7 @@ commons-collections-3.2.2.jar commons-collections4-4.4.jar commons-compiler-3.0.9.jar commons-compiler-3.1.6.jar -commons-compress-1.18.jar -commons-compress-1.19.jar commons-compress-1.20.jar -commons-compress-1.21.jar -commons-compress-1.4.1.jar -commons-compress-1.8.1.jar commons-configuration-1.7.jar commons-configuration2-2.1.1.jar commons-crypto-1.0.0.jar From f51c633374304de1375cdab1fa4de6d3f8a4536d Mon Sep 17 00:00:00 2001 From: Hisoka Date: Fri, 9 Sep 2022 13:56:35 +0800 Subject: [PATCH 17/22] [Engine] [Starter] fix license --- seatunnel-dist/release-docs/LICENSE | 1 - tools/dependencies/known-dependencies.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/seatunnel-dist/release-docs/LICENSE b/seatunnel-dist/release-docs/LICENSE index 8fd55eb617e..fb6b6359f90 100644 --- a/seatunnel-dist/release-docs/LICENSE +++ b/seatunnel-dist/release-docs/LICENSE @@ -1207,7 +1207,6 @@ The following components are provided under the Public Domain License. See proje The text of each license is also included at licenses/LICENSE-[project].txt. (Public Domain) AOP alliance (aopalliance:aopalliance:1.0 - http://aopalliance.sourceforge.net) - (Public Domain) XZ for Java (org.tukaani:xz:1.0 - http://tukaani.org/xz/java.html) (Public Domain) XZ for Java (org.tukaani:xz:1.5 - http://tukaani.org/xz/java.html) (Public Domain) XZ for Java (org.tukaani:xz:1.8 - https://tukaani.org/xz/java.html) (Public Domain, per Creative Commons CC0) HdrHistogram (org.hdrhistogram:HdrHistogram:2.1.9 - http://hdrhistogram.github.io/HdrHistogram/) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index f662660a98f..0ddee1d0846 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -717,7 +717,6 @@ xercesImpl-2.9.1.jar xml-apis-1.3.04.jar xmlbeans-3.1.0.jar xmlenc-0.52.jar -xz-1.0.jar xz-1.5.jar xz-1.8.jar zkclient-0.3.jar From 8e6d6411436a0514781e72795624afb654623920 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Fri, 9 Sep 2022 18:50:00 +0800 Subject: [PATCH 18/22] [Engine] [E2E] Change shell name in test container --- .../java/org/apache/seatunnel/e2e/flink/FlinkContainer.java | 2 +- .../java/org/apache/seatunnel/e2e/spark/SparkContainer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java index 0ffa3163480..6666bd5efb7 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/flink/FlinkContainer.java @@ -53,7 +53,7 @@ public abstract class FlinkContainer { protected GenericContainer jobManager; protected GenericContainer taskManager; private static final Path PROJECT_ROOT_PATH = Paths.get(System.getProperty("user.dir")).getParent().getParent(); - private static final String SEATUNNEL_FLINK_BIN = "start-seatunnel-flink-new-connector.sh"; + private static final String SEATUNNEL_FLINK_BIN = "start-seatunnel-flink-connector-v2.sh"; private static final String SEATUNNEL_FLINK_JAR = "seatunnel-flink-starter.jar"; private static final String PLUGIN_MAPPING_FILE = "plugin-mapping.properties"; private static final String SEATUNNEL_HOME = "/tmp/flink/seatunnel"; diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java index de30eeb41d7..282a74c6af6 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/java/org/apache/seatunnel/e2e/spark/SparkContainer.java @@ -52,7 +52,7 @@ public abstract class SparkContainer { protected GenericContainer master; private static final Path PROJECT_ROOT_PATH = Paths.get(System.getProperty("user.dir")).getParent().getParent(); - private static final String SEATUNNEL_SPARK_BIN = "start-seatunnel-spark-new-connector.sh"; + private static final String SEATUNNEL_SPARK_BIN = "start-seatunnel-spark-connector-v2.sh"; private static final String SEATUNNEL_SPARK_JAR = "seatunnel-spark-starter.jar"; private static final String PLUGIN_MAPPING_FILE = "plugin-mapping.properties"; private static final String SEATUNNEL_HOME = "/tmp/spark/seatunnel"; From 122ae8b1133f79b7fe9e370d569abb0da0e65d5f Mon Sep 17 00:00:00 2001 From: Hisoka Date: Tue, 13 Sep 2022 11:36:24 +0800 Subject: [PATCH 19/22] [Engine] [E2E] Fix ci problems --- .../test/java/org/apache/seatunnel/engine/e2e/TestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java index 786a34ce63f..f44177e09f9 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/TestUtils.java @@ -37,7 +37,7 @@ public static String getClusterName(String testClassName) { public static void initPluginDir() { // copy connectors to project_root/connectors dir System.setProperty("SEATUNNEL_HOME", System.getProperty("user.dir") + - String.format("%s..%s..%s", File.separator, File.separator, File.separator)); + String.format("%s..%s..%s..%s", File.separator, File.separator, File.separator, File.separator)); File seatunnelRootDir = new File(System.getProperty("SEATUNNEL_HOME")); File connectorDir = new File(seatunnelRootDir + From 702da46bd0486580c45f3859a7910e35e2ee3cbc Mon Sep 17 00:00:00 2001 From: Hisoka Date: Tue, 13 Sep 2022 17:49:18 +0800 Subject: [PATCH 20/22] [Engine] [Committer] Add committer classloader --- .../task/operation/sink/SinkPrepareCommitOperation.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/operation/sink/SinkPrepareCommitOperation.java b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/operation/sink/SinkPrepareCommitOperation.java index 6d970ed53d5..2eb9bb57db2 100644 --- a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/operation/sink/SinkPrepareCommitOperation.java +++ b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/task/operation/sink/SinkPrepareCommitOperation.java @@ -71,8 +71,8 @@ public int getClassId() { public void run() throws Exception { SeaTunnelServer server = getService(); SinkAggregatedCommitterTask committerTask = server.getTaskExecutionService().getTask(taskLocation); - // TODO add classloader support with #2704 - committerTask.receivedWriterCommitInfo(barrier.getId(), SerializationUtils.deserialize(commitInfos)); + ClassLoader classLoader = server.getTaskExecutionService().getExecutionContext(taskLocation.getTaskGroupLocation()).getClassLoader(); + committerTask.receivedWriterCommitInfo(barrier.getId(), SerializationUtils.deserialize(commitInfos, classLoader)); committerTask.triggerBarrier(barrier); } } From cbc70de144c586379bb718ee3ec05d1bbafd3b5e Mon Sep 17 00:00:00 2001 From: Hisoka Date: Wed, 14 Sep 2022 20:10:51 +0800 Subject: [PATCH 21/22] [Starter] [Core] Change README.md --- seatunnel-core/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-core/README.md b/seatunnel-core/README.md index 05fdf0e15e1..7ba7c840f86 100644 --- a/seatunnel-core/README.md +++ b/seatunnel-core/README.md @@ -7,5 +7,5 @@ This module is the seatunnel job entrypoint. Seatunnel jobs are started by the b - seatunnel-core-flink-sql: The flink sql job starter. - seatunnel-core-spark: The spark job starter. - seatunnel-spark-starter: The spark job starter for connector-v2. -- seatunnel-spark-starter: The flink job starter for connector-v2. +- seatunnel-flink-starter: The flink job starter for connector-v2. - seatunnel-starter: The seatunnel engine job starter for connector-v2. \ No newline at end of file From 23680c9feb94c9cc8b821b029eb82e8df4a70af9 Mon Sep 17 00:00:00 2001 From: Hisoka Date: Thu, 15 Sep 2022 10:15:53 +0800 Subject: [PATCH 22/22] [Starter] [Core] Resolved conflict --- .../seatunnel/core/starter/spark/execution/SparkExecution.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java index ddcad59b01e..0ec9f66620a 100644 --- a/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java +++ b/seatunnel-core/seatunnel-spark-starter/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkExecution.java @@ -18,8 +18,6 @@ package org.apache.seatunnel.core.starter.spark.execution; import org.apache.seatunnel.api.common.JobContext; -import org.apache.seatunnel.core.starter.config.EngineType; -import org.apache.seatunnel.core.starter.config.EnvironmentFactory; import org.apache.seatunnel.core.starter.exception.TaskExecuteException; import org.apache.seatunnel.core.starter.spark.config.SparkEnvironmentFactory; import org.apache.seatunnel.spark.SparkEnvironment;