添加打包更换etl-plugins 模块名为connectors

harbby · Aug 20, 2018 · 6b60afa · 6b60afa
1 parent cd6f5d2
commit 6b60afa
Show file tree

Hide file tree

Showing 17 changed files with 64 additions and 29 deletions.
diff --git a/settings.gradle b/settings.gradle
@@ -16,9 +16,9 @@ project(':sylph-runners:spark').name = 'sylph-runner-spark'
 
 //----
 include 'sylph-etl-api'
-include 'sylph-etl-plugins'
-include 'sylph-etl-plugins:flink-node-plugin'
-include 'sylph-etl-plugins:spark-node-plugin'
+include 'sylph-connectors'
+include 'sylph-connectors:flink-node-plugin'
+include 'sylph-connectors:spark-node-plugin'
 
 //----
 include 'sylph-dist'

diff --git a/sylph-etl-plugins/build.gradle → sylph-connectors/build.gradle b/sylph-etl-plugins/build.gradle → sylph-connectors/build.gradle
@@ -1,4 +1,4 @@
-
+evaluationDependsOn(':sylph-dist')
 subprojects {
     apply plugin: 'com.github.harbby.gradle.serviceloader'
     serviceLoader {

diff --git a/...tl-plugins/flink-node-plugin/build.gradle → ...connectors/flink-node-plugin/build.gradle b/...tl-plugins/flink-node-plugin/build.gradle → ...connectors/flink-node-plugin/build.gradle
diff --git a/...l/sylph/plugins/flink/sink/MysqlSink.java → ...l/sylph/plugins/flink/sink/MysqlSink.java b/...l/sylph/plugins/flink/sink/MysqlSink.java → ...l/sylph/plugins/flink/sink/MysqlSink.java
diff --git a/.../sylph/plugins/flink/sink/PrintSink.scala → .../sylph/plugins/flink/sink/PrintSink.scala b/.../sylph/plugins/flink/sink/PrintSink.scala → .../sylph/plugins/flink/sink/PrintSink.scala
diff --git a/...ph/plugins/flink/source/FlinkEnvUtil.java → ...ph/plugins/flink/source/FlinkEnvUtil.java b/...ph/plugins/flink/source/FlinkEnvUtil.java → ...ph/plugins/flink/source/FlinkEnvUtil.java
diff --git a/...lph/plugins/flink/source/KafkaSource.java → ...lph/plugins/flink/source/KafkaSource.java b/...lph/plugins/flink/source/KafkaSource.java → ...lph/plugins/flink/source/KafkaSource.java
diff --git a/...lph/plugins/flink/source/TestSource.scala → ...lph/plugins/flink/source/TestSource.scala b/...lph/plugins/flink/source/TestSource.scala → ...lph/plugins/flink/source/TestSource.scala
diff --git a/...h/plugins/flink/transform/SqlWindow.scala → ...h/plugins/flink/transform/SqlWindow.scala b/...h/plugins/flink/transform/SqlWindow.scala → ...h/plugins/flink/transform/SqlWindow.scala
diff --git a/...h/plugins/flink/transform/TestTrans.scala → ...h/plugins/flink/transform/TestTrans.scala b/...h/plugins/flink/transform/TestTrans.scala → ...h/plugins/flink/transform/TestTrans.scala
diff --git a/...tl-plugins/spark-node-plugin/build.gradle → ...connectors/spark-node-plugin/build.gradle b/...tl-plugins/spark-node-plugin/build.gradle → ...connectors/spark-node-plugin/build.gradle
diff --git a/....plugins.spark/source/MyKafkaSource.scala → ....plugins.spark/source/MyKafkaSource.scala b/....plugins.spark/source/MyKafkaSource.scala → ....plugins.spark/source/MyKafkaSource.scala
diff --git a/...h.plugins.spark/source/SocketSource.scala → ...h.plugins.spark/source/SocketSource.scala b/...h.plugins.spark/source/SocketSource.scala → ...h.plugins.spark/source/SocketSource.scala
diff --git a/sylph-dist/build.gradle b/sylph-dist/build.gradle
@@ -13,8 +13,42 @@ task copyFiles(type: Copy,dependsOn: copyLibs){
 
 assemble.dependsOn copyFiles
 
-task dist(){
-//  from(configurations.runtime) {
-//    into tarpath+"/lib"
-//  }
+task dist(type: Tar) {
+  baseName = rootProject.getName()
+  classifier = 'bin'
+  extension = 'tgz'
+  compression = Compression.GZIP
+  def tarpath = baseName+"-${rootProject.version}"
+
+  from(configurations.runtime) {
+    into tarpath+"/lib"
+  }
+
+  from(project.files('src/bin')) {
+    into(tarpath+"/bin")
+    fileMode = 0755
+  }
+  from(project.files('src/etc')) {
+    into(tarpath+"/etc")
+  }
+
+  from(project.files('src/jobs')) {
+    into(tarpath+"/jobs")
+  }
+
+  from(project.files('src/webapps')) {
+    into(tarpath+"/webapps")
+  }
+
+  from(project.files('build/modules')){
+    into tarpath+"/modules"
+  }
+
+  from(project.files('build/etl-plugins')) {
+    into(tarpath+"/etl-plugins")
+  }
+}
+
+artifacts {
+  'default' dist
 }
diff --git a/sylph-dist/src/webapps/index.html b/sylph-dist/src/webapps/index.html
@@ -1,6 +1,6 @@
 <html lang="en">
 <meta charset="UTF-8">
-<title>任务管理</title>
+<title>JobManager</title>
 <link href="./css/bootstrap.min.css" rel="stylesheet" type="text/css"/>
 <script type="text/javascript" src="./libs/jquery-1.11.1.min.js"></script>
 <script type="text/javascript" src="./js/list.js"></script>
@@ -28,32 +28,33 @@
 <body>
 
 
-<h2>任务管理</h2>
+<h2>JobManager</h2>
 
-<form method="post" action="/_sys/job_manger" enctype="multipart/form-data">
-    <div style="width: 500px; margin-bottom: 50px">
-        <div class="form-group">
-            <label>上传任务</label>
-            <input type="file" name="file" class="form-control" />
-        </div>
-        <span>
-            <input type="submit" class="btn btn-primary" value="点击上传" />
-        </span>
-    </div>
-</form>
+<!--<form method="post" action="/_sys/job_manger" enctype="multipart/form-data">-->
+<!--<div style="width: 500px; margin-bottom: 50px">-->
+<!--<div class="form-group">-->
+<!--<label>上传任务</label>-->
+<!--<input type="file" name="file" class="form-control" />-->
+<!--</div>-->
+<!--<span>-->
+<!--<input type="submit" class="btn btn-primary" value="点击上传" />-->
+<!--</span>-->
+<!--</div>-->
+<!--</form>-->
 
 <div class="row">
     <div class="col-md-8"><label>插件列表</label></div>
     <div class="col-md-4">
         <button class="btn btn-primary refresh_all">刷新</button>
-        <button class="btn btn-primary" onclick='window.location.href="etl.html"'>新建任务</button>
+        <button class="btn btn-primary" onclick='window.location.href="etl.html"'>新建etl任务</button>
+        <button class="btn btn-primary" onclick='window.location.href="stream_sql.html"'>新建StreamSql任务</button>
     </div>
 </div>
 
 <div class="row" id="rowHead">
     <div class="col-md-2">任务id</div>
     <div class="col-md-3">AppId</div>
-	<div class="col-md-1">类型</div>
+    <div class="col-md-1">类型</div>
     <div class="col-md-2">创建时间</div>
     <div class="col-md-1">状态</div>
     <div class="col-md-3">操作</div>

diff --git a/sylph-main/src/main/java/ideal.sylph/main/service/LocalJobStore.java b/sylph-main/src/main/java/ideal.sylph/main/service/LocalJobStore.java
@@ -59,8 +59,8 @@ public void saveJob(@Nonnull Job job)
         job.getId();
         try {
             Flow flow = job.getFlow();
-            File yaml = new File(jobDir, "job.yaml");
-            File typeFile = new File(jobDir, "type.job");
+            File yaml = new File(jobDir, "job.flow");
+            File typeFile = new File(jobDir, "job.type");
 
             //TODO: save?
             String jobType = job.getActuatorName();
@@ -108,17 +108,17 @@ public void loadJobs()
                 .parallel()
                 .forEach(jobDir -> {
                     try {
-                        final File typeFile = new File(jobDir, "type.job");
+                        final File typeFile = new File(jobDir, "job.type");
                         checkArgument(typeFile.exists() && typeFile.isFile(), typeFile + " is not exists or isDirectory");
                         Map<String, String> jobProps = loadProperties(typeFile);
                         String jobType = requireNonNull(jobProps.get("type"), "jobProps arg type is null");
                         try {
-                            byte[] flowBytes = Files.readAllBytes(Paths.get(new File(jobDir, "job.yaml").toURI()));
+                            byte[] flowBytes = Files.readAllBytes(Paths.get(new File(jobDir, "job.flow").toURI()));
                             Job job = runnerManger.formJobWithFlow(jobDir.getName(), flowBytes, jobType);
                             jobs.put(job.getId(), job);
                         }
                         catch (IOException e) {
-                            throw new SylphException(JOB_BUILD_ERROR, "loadding job " + jobDir + " job.yaml fail", e);
+                            throw new SylphException(JOB_BUILD_ERROR, "loadding job " + jobDir + " job.flow fail", e);
                         }
                     }
                     catch (Exception e) {

diff --git a/sylph-runners/flink/build.gradle b/sylph-runners/flink/build.gradle
@@ -8,7 +8,7 @@ dependencies {
   compileOnly(group: 'org.apache.flink', name: 'flink-streaming-java_2.11', version: deps.flink) {
     exclude(module: 'slf4j-api')
   }
-  compile group: 'org.apache.flink', name: 'flink-shaded-hadoop2', version: deps.flink
+  compileOnly group: 'org.apache.flink', name: 'flink-shaded-hadoop2', version: deps.flink
 
   compileOnly(group: 'org.apache.flink', name: 'flink-streaming-scala_2.11', version: deps.flink) {
     exclude(module: 'slf4j-api')