From 6b60afa1ea29a1430e111012ce27d3eccb6ff7f6 Mon Sep 17 00:00:00 2001 From: ideal <125553253@qq.com> Date: Mon, 20 Aug 2018 12:07:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=89=93=E5=8C=85=20?= =?UTF-8?q?=E6=9B=B4=E6=8D=A2etl-plugins=20=E6=A8=A1=E5=9D=97=E5=90=8D?= =?UTF-8?q?=E4=B8=BAconnectors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- settings.gradle | 6 +-- .../build.gradle | 2 +- .../flink-node-plugin/build.gradle | 0 .../sylph/plugins/flink/sink/MysqlSink.java | 0 .../sylph/plugins/flink/sink/PrintSink.scala | 0 .../plugins/flink/source/FlinkEnvUtil.java | 0 .../plugins/flink/source/KafkaSource.java | 0 .../plugins/flink/source/TestSource.scala | 0 .../plugins/flink/transform/SqlWindow.scala | 0 .../plugins/flink/transform/TestTrans.scala | 0 .../spark-node-plugin/build.gradle | 0 .../source/MyKafkaSource.scala | 0 .../source/SocketSource.scala | 0 sylph-dist/build.gradle | 42 +++++++++++++++++-- sylph-dist/src/webapps/index.html | 31 +++++++------- .../main/service/LocalJobStore.java | 10 ++--- sylph-runners/flink/build.gradle | 2 +- 17 files changed, 64 insertions(+), 29 deletions(-) rename {sylph-etl-plugins => sylph-connectors}/build.gradle (95%) rename {sylph-etl-plugins => sylph-connectors}/flink-node-plugin/build.gradle (100%) rename {sylph-etl-plugins => sylph-connectors}/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/MysqlSink.java (100%) rename {sylph-etl-plugins => sylph-connectors}/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/PrintSink.scala (100%) rename {sylph-etl-plugins => sylph-connectors}/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/FlinkEnvUtil.java (100%) rename {sylph-etl-plugins => sylph-connectors}/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/KafkaSource.java (100%) rename {sylph-etl-plugins => sylph-connectors}/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/TestSource.scala (100%) rename {sylph-etl-plugins => sylph-connectors}/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/SqlWindow.scala (100%) rename {sylph-etl-plugins => sylph-connectors}/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/TestTrans.scala (100%) rename {sylph-etl-plugins => sylph-connectors}/spark-node-plugin/build.gradle (100%) rename {sylph-etl-plugins => sylph-connectors}/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/MyKafkaSource.scala (100%) rename {sylph-etl-plugins => sylph-connectors}/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/SocketSource.scala (100%) diff --git a/settings.gradle b/settings.gradle index 041a4bc3d..389a5e72c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,9 +16,9 @@ project(':sylph-runners:spark').name = 'sylph-runner-spark' //---- include 'sylph-etl-api' -include 'sylph-etl-plugins' -include 'sylph-etl-plugins:flink-node-plugin' -include 'sylph-etl-plugins:spark-node-plugin' +include 'sylph-connectors' +include 'sylph-connectors:flink-node-plugin' +include 'sylph-connectors:spark-node-plugin' //---- include 'sylph-dist' diff --git a/sylph-etl-plugins/build.gradle b/sylph-connectors/build.gradle similarity index 95% rename from sylph-etl-plugins/build.gradle rename to sylph-connectors/build.gradle index 12bdfe6e8..a06dd211c 100644 --- a/sylph-etl-plugins/build.gradle +++ b/sylph-connectors/build.gradle @@ -1,4 +1,4 @@ - +evaluationDependsOn(':sylph-dist') subprojects { apply plugin: 'com.github.harbby.gradle.serviceloader' serviceLoader { diff --git a/sylph-etl-plugins/flink-node-plugin/build.gradle b/sylph-connectors/flink-node-plugin/build.gradle similarity index 100% rename from sylph-etl-plugins/flink-node-plugin/build.gradle rename to sylph-connectors/flink-node-plugin/build.gradle diff --git a/sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/MysqlSink.java b/sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/MysqlSink.java similarity index 100% rename from sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/MysqlSink.java rename to sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/MysqlSink.java diff --git a/sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/PrintSink.scala b/sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/PrintSink.scala similarity index 100% rename from sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/PrintSink.scala rename to sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/sink/PrintSink.scala diff --git a/sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/FlinkEnvUtil.java b/sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/FlinkEnvUtil.java similarity index 100% rename from sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/FlinkEnvUtil.java rename to sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/FlinkEnvUtil.java diff --git a/sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/KafkaSource.java b/sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/KafkaSource.java similarity index 100% rename from sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/KafkaSource.java rename to sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/KafkaSource.java diff --git a/sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/TestSource.scala b/sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/TestSource.scala similarity index 100% rename from sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/TestSource.scala rename to sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/source/TestSource.scala diff --git a/sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/SqlWindow.scala b/sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/SqlWindow.scala similarity index 100% rename from sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/SqlWindow.scala rename to sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/SqlWindow.scala diff --git a/sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/TestTrans.scala b/sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/TestTrans.scala similarity index 100% rename from sylph-etl-plugins/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/TestTrans.scala rename to sylph-connectors/flink-node-plugin/src/main/scala/ideal/sylph/plugins/flink/transform/TestTrans.scala diff --git a/sylph-etl-plugins/spark-node-plugin/build.gradle b/sylph-connectors/spark-node-plugin/build.gradle similarity index 100% rename from sylph-etl-plugins/spark-node-plugin/build.gradle rename to sylph-connectors/spark-node-plugin/build.gradle diff --git a/sylph-etl-plugins/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/MyKafkaSource.scala b/sylph-connectors/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/MyKafkaSource.scala similarity index 100% rename from sylph-etl-plugins/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/MyKafkaSource.scala rename to sylph-connectors/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/MyKafkaSource.scala diff --git a/sylph-etl-plugins/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/SocketSource.scala b/sylph-connectors/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/SocketSource.scala similarity index 100% rename from sylph-etl-plugins/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/SocketSource.scala rename to sylph-connectors/spark-node-plugin/src/main/scala/ideal.sylph.plugins.spark/source/SocketSource.scala diff --git a/sylph-dist/build.gradle b/sylph-dist/build.gradle index ea5d1b7b4..bc334d97f 100644 --- a/sylph-dist/build.gradle +++ b/sylph-dist/build.gradle @@ -13,8 +13,42 @@ task copyFiles(type: Copy,dependsOn: copyLibs){ assemble.dependsOn copyFiles -task dist(){ -// from(configurations.runtime) { -// into tarpath+"/lib" -// } +task dist(type: Tar) { + baseName = rootProject.getName() + classifier = 'bin' + extension = 'tgz' + compression = Compression.GZIP + def tarpath = baseName+"-${rootProject.version}" + + from(configurations.runtime) { + into tarpath+"/lib" + } + + from(project.files('src/bin')) { + into(tarpath+"/bin") + fileMode = 0755 + } + from(project.files('src/etc')) { + into(tarpath+"/etc") + } + + from(project.files('src/jobs')) { + into(tarpath+"/jobs") + } + + from(project.files('src/webapps')) { + into(tarpath+"/webapps") + } + + from(project.files('build/modules')){ + into tarpath+"/modules" + } + + from(project.files('build/etl-plugins')) { + into(tarpath+"/etl-plugins") + } +} + +artifacts { + 'default' dist } \ No newline at end of file diff --git a/sylph-dist/src/webapps/index.html b/sylph-dist/src/webapps/index.html index 661749259..405822d4d 100644 --- a/sylph-dist/src/webapps/index.html +++ b/sylph-dist/src/webapps/index.html @@ -1,6 +1,6 @@ -任务管理 +JobManager @@ -28,32 +28,33 @@ -

任务管理

+

JobManager

-
-
-
- - -
- - - -
-
+ + + + + + + + + + +
- + +
任务id
AppId
-
类型
+
类型
创建时间
状态
操作
diff --git a/sylph-main/src/main/java/ideal.sylph/main/service/LocalJobStore.java b/sylph-main/src/main/java/ideal.sylph/main/service/LocalJobStore.java index 48c8a4438..188d78599 100644 --- a/sylph-main/src/main/java/ideal.sylph/main/service/LocalJobStore.java +++ b/sylph-main/src/main/java/ideal.sylph/main/service/LocalJobStore.java @@ -59,8 +59,8 @@ public void saveJob(@Nonnull Job job) job.getId(); try { Flow flow = job.getFlow(); - File yaml = new File(jobDir, "job.yaml"); - File typeFile = new File(jobDir, "type.job"); + File yaml = new File(jobDir, "job.flow"); + File typeFile = new File(jobDir, "job.type"); //TODO: save? String jobType = job.getActuatorName(); @@ -108,17 +108,17 @@ public void loadJobs() .parallel() .forEach(jobDir -> { try { - final File typeFile = new File(jobDir, "type.job"); + final File typeFile = new File(jobDir, "job.type"); checkArgument(typeFile.exists() && typeFile.isFile(), typeFile + " is not exists or isDirectory"); Map jobProps = loadProperties(typeFile); String jobType = requireNonNull(jobProps.get("type"), "jobProps arg type is null"); try { - byte[] flowBytes = Files.readAllBytes(Paths.get(new File(jobDir, "job.yaml").toURI())); + byte[] flowBytes = Files.readAllBytes(Paths.get(new File(jobDir, "job.flow").toURI())); Job job = runnerManger.formJobWithFlow(jobDir.getName(), flowBytes, jobType); jobs.put(job.getId(), job); } catch (IOException e) { - throw new SylphException(JOB_BUILD_ERROR, "loadding job " + jobDir + " job.yaml fail", e); + throw new SylphException(JOB_BUILD_ERROR, "loadding job " + jobDir + " job.flow fail", e); } } catch (Exception e) { diff --git a/sylph-runners/flink/build.gradle b/sylph-runners/flink/build.gradle index 32187a8d5..bd332b489 100644 --- a/sylph-runners/flink/build.gradle +++ b/sylph-runners/flink/build.gradle @@ -8,7 +8,7 @@ dependencies { compileOnly(group: 'org.apache.flink', name: 'flink-streaming-java_2.11', version: deps.flink) { exclude(module: 'slf4j-api') } - compile group: 'org.apache.flink', name: 'flink-shaded-hadoop2', version: deps.flink + compileOnly group: 'org.apache.flink', name: 'flink-shaded-hadoop2', version: deps.flink compileOnly(group: 'org.apache.flink', name: 'flink-streaming-scala_2.11', version: deps.flink) { exclude(module: 'slf4j-api')