From d80d8324335609df5cdd08257e74360bf659e584 Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Tue, 14 Mar 2023 13:41:12 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E8=B0=83=E6=95=B4=20JVM=20=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E9=80=82=E5=BA=94=E9=AB=98=E7=89=88=E6=9C=AC=20JDK?= =?UTF-8?q?=EF=BC=8C=E7=A7=BB=E9=99=A4=E5=86=85=E5=AD=98=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E5=8F=82=E6=95=B0=EF=BC=8C=E4=B8=8D=E5=88=86=E4=B8=8D=E9=80=82?= =?UTF-8?q?=E7=94=A8=EF=BC=8C=E7=A7=BB=E9=99=A4=20DS=20copy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/datasophon-api.sh | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/datasophon-api/src/main/resources/datasophon-api.sh b/datasophon-api/src/main/resources/datasophon-api.sh index 1dd2250e..d4bb0390 100644 --- a/datasophon-api/src/main/resources/datasophon-api.sh +++ b/datasophon-api/src/main/resources/datasophon-api.sh @@ -31,12 +31,35 @@ shift echo "Begin $startStop $command......" -BIN_DIR=`dirname $0` -BIN_DIR=`cd "$BIN_DIR"; pwd` -DDH_HOME=$BIN_DIR/.. - source /etc/profile +SCRIPT="$0" +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# some Java parameters +JAVA=`which java 2>/dev/null` +if [[ $JAVA_HOME != "" ]]; then + JAVA=$JAVA_HOME/bin/java +fi +if test -z "$JAVA"; then + echo "No java found in the PATH. Please set JAVA_HOME." + exit 1 +fi + +BIN_DIR=`dirname "$SCRIPT"`/.. +BIN_DIR=`cd "$BIN_DIR"; pwd` +export DDH_HOME=$BIN_DIR + export JAVA_HOME=$JAVA_HOME #export JAVA_HOME=/opt/soft/jdk export HOSTNAME=`hostname` @@ -46,7 +69,7 @@ export DDH_LOG_DIR=$DDH_HOME/logs export DDH_CONF_DIR=$DDH_HOME/conf export DDH_LIB_JARS=$DDH_HOME/lib/* -export DDH_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DDH_OPTS" +export DDH_OPTS="-server $DDH_OPTS" export STOP_TIMEOUT=5 if [ ! -d "$DDH_LOG_DIR" ]; then @@ -86,8 +109,8 @@ case $startStop in exec_command="$LOG_FILE $DDH_OPTS -classpath $DDH_CONF_DIR:$DDH_LIB_JARS $CLASS" - echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" - nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & + echo "nohup $JAVA $exec_command > $log 2>&1 &" + nohup $JAVA $exec_command > $log 2>&1 & echo $! > $pid ;; @@ -140,8 +163,8 @@ case $startStop in exec_command="$LOG_FILE $DDH_OPTS -classpath $DDH_CONF_DIR:$DDH_LIB_JARS $CLASS" - echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" - nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & + echo "nohup $JAVA $exec_command > $log 2>&1 &" + nohup $JAVA $exec_command > $log 2>&1 & echo $! > $pid ;; (*) From ae2831fa89fdcef891e50e53c70755676df35192 Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Wed, 15 Mar 2023 20:47:13 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E8=B0=83=E6=95=B4ui=E7=9A=84=E6=89=93?= =?UTF-8?q?=E5=8C=85=EF=BC=8C=E4=BD=BF=E6=97=A0=E9=9C=80copy=20dist?= =?UTF-8?q?=E5=88=B0api=20resource=EF=BC=9B=E5=A2=9E=E5=8A=A0versions-mave?= =?UTF-8?q?n-plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- datasophon-api/pom.xml | 57 ++++---- .../src/main/resources/assembly.xml | 44 ++++-- datasophon-ui/pom.xml | 104 +++++++++------ datasophon-worker/pom.xml | 125 ++++++++---------- .../src/main/resources/assembly.xml | 32 ++--- pom.xml | 45 ++++++- 6 files changed, 226 insertions(+), 181 deletions(-) diff --git a/datasophon-api/pom.xml b/datasophon-api/pom.xml index 9a9ad06b..9c859d50 100644 --- a/datasophon-api/pom.xml +++ b/datasophon-api/pom.xml @@ -15,16 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - + 4.0.0 - datasophon com.datasophon + datasophon 1.1.0 - 4.0.0 datasophon-api @@ -43,8 +41,8 @@ spring-boot-starter-tomcat - log4j-to-slf4j org.apache.logging.log4j + log4j-to-slf4j @@ -95,12 +93,18 @@ com.datasophon datasophon-dao - 1.1.0 + ${project.version} com.datasophon datasophon-common - 1.1.0 + ${project.version} + + + + com.datasophon + datasophon-ui + ${project.version} com.typesafe.akka @@ -115,7 +119,6 @@ akka-persistence_2.11 - org.apache.sshd sshd-core @@ -131,60 +134,46 @@ test - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-lib - package - - copy-dependencies - - - target/lib - false - false - runtime - - - - - maven-assembly-plugin + 3.3.0 false datasophon-manager-${project.version} - target/ + ${project.basedir}/target/ - src/main/resources/assembly.xml + ${project.basedir}/src/main/resources/assembly.xml + + + + + make-assembly - - package single + + package - \ No newline at end of file + diff --git a/datasophon-api/src/main/resources/assembly.xml b/datasophon-api/src/main/resources/assembly.xml index b3ef3b71..b8dafd07 100644 --- a/datasophon-api/src/main/resources/assembly.xml +++ b/datasophon-api/src/main/resources/assembly.xml @@ -24,6 +24,19 @@ tar.gz true + + + true + lib + 0755 + 0755 + + + *:log4j + *:slf4j-log4j12 + + + @@ -77,8 +90,10 @@ *.sh 0755 + 0755 true bin + unix @@ -90,30 +105,35 @@ ${basedir}/src/main/resources/jmx jmx + 0755 + 0755 ${basedir}/../datasophon-api/src/main/resources/sql sql + 0755 + 0755 - - - ${project.build.directory}/lib - lib - + ${basedir}/../datasophon-ui/dist + /conf/front/views - *.jar + *.html + *.ico + *.js + 0755 + 0755 - ${project.build.directory} - lib - + ${basedir}/../datasophon-ui/dist + /conf/front/static/resources/bundle-main + 0755 + 0755 - datasophon-api-${project.version}.jar + css/** + static/** - - diff --git a/datasophon-ui/pom.xml b/datasophon-ui/pom.xml index 9cd97758..d4cc681c 100644 --- a/datasophon-ui/pom.xml +++ b/datasophon-ui/pom.xml @@ -15,33 +15,61 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - + 4.0.0 - datasophon com.datasophon + datasophon 1.1.0 - 4.0.0 datasophon-ui v12.16.3 6.14.4 - node_modules - dist .deployed + dist + node_modules + + org.apache.maven.plugins + maven-clean-plugin + 3.0.0 + + + false + + false + + + + ${basedir}/dist + + false + + true + + + + + + com.github.eirslett frontend-maven-plugin 1.10.3 + + ${build.node.version} + ${build.npm.version} + + https://npm.taobao.org/mirrors/node/ + https://registry.npm.taobao.org/npm/-/ + install node and npm @@ -57,8 +85,7 @@ generate-resources - install --registry=https://registry.npm.taobao.org --sass-binary-site=https://npm.taobao.org/mirrors/node-sass - + install --registry=https://registry.npm.taobao.org --sass-binary-site=https://npm.taobao.org/mirrors/node-sass @@ -66,29 +93,24 @@ npm - install + package run build - - npm run deploy - - npm - - install - - run deploy - - + - - ${build.node.version} - ${build.npm.version} - - https://npm.taobao.org/mirrors/node/ - https://registry.npm.taobao.org/npm/-/ - @@ -101,21 +123,21 @@ - win + /C brunch + /C mkdir + /C npm + /C rmdir /S /Q + /C \ + win cmd ${basedir}\gzip-content.cmd - /C brunch - node.exe cmd - /C mkdir cmd - /C npm cmd - /C rmdir /S /Q cmd cmd - /C + node.exe @@ -126,21 +148,21 @@ - linux + + + + -rf + / + linux brunch gzip - - node mkdir - npm - rm - -rf sh sh - + node diff --git a/datasophon-worker/pom.xml b/datasophon-worker/pom.xml index 3aebe882..58125e16 100644 --- a/datasophon-worker/pom.xml +++ b/datasophon-worker/pom.xml @@ -15,16 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - + 4.0.0 - datasophon com.datasophon + datasophon 1.1.0 - 4.0.0 datasophon-worker @@ -34,52 +32,52 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - + + + + com.typesafe.akka akka-actor_2.11 @@ -99,7 +97,7 @@ com.datasophon datasophon-common - 1.1.0 + ${project.version} org.freemarker @@ -126,35 +124,18 @@ org.apache.maven.plugins - maven-dependency-plugin - - - copy-lib - package - - copy-dependencies - - - target/lib - false - false - runtime - - - - - maven-assembly-plugin + 3.3.0 false datasophon-worker - target/ + ${project.basedir}/target/ - src/main/resources/assembly.xml + ${project.basedir}/src/main/resources/assembly.xml @@ -162,16 +143,16 @@ make-assembly - - package single + + package - \ No newline at end of file + diff --git a/datasophon-worker/src/main/resources/assembly.xml b/datasophon-worker/src/main/resources/assembly.xml index b2504a9c..9cb9b2eb 100644 --- a/datasophon-worker/src/main/resources/assembly.xml +++ b/datasophon-worker/src/main/resources/assembly.xml @@ -24,6 +24,19 @@ tar.gz true + + + true + lib + 0755 + 0755 + + + *:log4j + *:slf4j-log4j12 + + + @@ -135,24 +148,5 @@ unix /node/arm - - - - ${project.build.directory}/lib - lib - - - *.jar - - - - ${project.build.directory} - lib - - - datasophon-worker-${project.version}.jar - - - diff --git a/pom.xml b/pom.xml index 94f40949..a9df02f2 100644 --- a/pom.xml +++ b/pom.xml @@ -15,9 +15,7 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - 4.0.0 @@ -60,6 +58,8 @@ 2.27.1 2.1.18.RELEASE 5.1.19.RELEASE + + 2.14.2 @@ -182,6 +182,39 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + true + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.2 + + ${java.version} + ${java.version} + UTF-8 + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + + jar-no-fork + + package + + + com.diffplug.spotless spotless-maven-plugin @@ -244,6 +277,12 @@ + + + org.codehaus.mojo + versions-maven-plugin + ${versions-maven-plugin.version} + From d09a09062aa98ac91a3abdb6cd3194b4acd42bbf Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Thu, 20 Apr 2023 08:54:31 +0800 Subject: [PATCH 3/9] =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=BC=98=E5=85=88?= =?UTF-8?q?=E4=BB=8Eworker=20templates=E4=B8=8B=E5=8A=A0=E8=BD=BD=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=A8=A1=E7=89=88=EF=BC=8C=E5=85=B6=E6=AC=A1=E4=BB=8E?= =?UTF-8?q?component=E6=A0=B9=E7=9B=AE=E5=BD=95=E4=B8=8B=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../handler/ConfigureServiceHandler.java | 15 +++++- .../worker/utils/FreemakerUtils.java | 42 ++++++++++++++- .../worker/test/MutilTemplateFileTest.java | 52 +++++++++++++++++++ 3 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 datasophon-worker/src/main/test/com/datasophon/worker/test/MutilTemplateFileTest.java diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/handler/ConfigureServiceHandler.java b/datasophon-worker/src/main/java/com/datasophon/worker/handler/ConfigureServiceHandler.java index f5aac025..e7b07679 100644 --- a/datasophon-worker/src/main/java/com/datasophon/worker/handler/ConfigureServiceHandler.java +++ b/datasophon-worker/src/main/java/com/datasophon/worker/handler/ConfigureServiceHandler.java @@ -28,6 +28,7 @@ import org.apache.commons.lang.StringUtils; +import java.io.File; import java.net.InetAddress; import java.util.*; @@ -126,7 +127,17 @@ public ExecResult configure(Map> cofigFileMap, } configs.addAll(customConfList); if (configs.size() > 0) { - FreemakerUtils.generateConfigFile(generators, configs, decompressPackageName); + // extra app, package: META, templates + File extTemplateDir = + new File(Constants.INSTALL_PATH + File.separator + decompressPackageName, "templates"); + if (extTemplateDir.exists() && extTemplateDir.isDirectory()) { + // 3rd app, load ext templates + logger.info("add ext app template path: {} to loader path.", extTemplateDir.getAbsolutePath()); + FreemakerUtils.generateConfigFile(generators, configs, decompressPackageName, + extTemplateDir.getAbsolutePath()); + } else { + FreemakerUtils.generateConfigFile(generators, configs, decompressPackageName); + } } execResult.setExecOut("configure success"); logger.info("configure success"); @@ -137,7 +148,7 @@ public ExecResult configure(Map> cofigFileMap, execResult.setExecResult(true); } catch (Exception e) { execResult.setExecErrOut(e.getMessage()); - e.printStackTrace(); + logger.error("load app config template error!", e); } return execResult; } diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/utils/FreemakerUtils.java b/datasophon-worker/src/main/java/com/datasophon/worker/utils/FreemakerUtils.java index ccb81203..e6fc3956 100644 --- a/datasophon-worker/src/main/java/com/datasophon/worker/utils/FreemakerUtils.java +++ b/datasophon-worker/src/main/java/com/datasophon/worker/utils/FreemakerUtils.java @@ -22,11 +22,18 @@ import com.datasophon.common.model.Generators; import com.datasophon.common.model.ServiceConfig; +import freemarker.cache.ClassTemplateLoader; +import freemarker.cache.FileTemplateLoader; +import freemarker.cache.MultiTemplateLoader; +import freemarker.cache.TemplateLoader; import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; +import org.apache.commons.lang.StringUtils; + import java.io.*; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,14 +49,44 @@ public class FreemakerUtils { private static final Logger logger = LoggerFactory.getLogger(FreemakerUtils.class); - public static void generateConfigFile(Generators generators, List configs, + public static void generateConfigFile(Generators generators, + List configs, String decompressPackageName) throws IOException, TemplateException { + generateConfigFile(generators, configs, decompressPackageName, null); + } + + /** + * + * 支持 从附加的目录加载 模版 + * + * @param generators + * @param configs + * @param decompressPackageName + * @param extPath + * @throws IOException + * @throws TemplateException + */ + public static void generateConfigFile(Generators generators, + List configs, + String decompressPackageName, + String extPath) throws IOException, TemplateException { // 1.加载模板 // 创建核心配置对象 Configuration config = new Configuration(Configuration.getVersion()); // 设置加载的目录 // ""代表当前包 - config.setClassForTemplateLoading(FreemakerUtils.class, "/templates"); + // config.setClassForTemplateLoading(FreemakerUtils.class, "/templates"); + + /* + * 第三方的程序包，pacakge 下应包含：META/template 和其他应用必须的文件 + */ + List loaderList = new ArrayList<>(); + loaderList.add(new ClassTemplateLoader(FreemakerUtils.class, "/templates")); + if (StringUtils.isNotBlank(extPath) && new File(extPath).exists()) { + // 如果 三方的 package 中存在 templates 模版，则直接加载 + loaderList.add(new FileTemplateLoader(new File(extPath))); + } + config.setTemplateLoader(new MultiTemplateLoader(loaderList.toArray(new TemplateLoader[0]))); Map data = new HashMap<>(); // 得到模板对象 @@ -76,6 +113,7 @@ public static void generateConfigFile(Generators generators, List .collect(Collectors.toMap(key -> key.getName(), value -> value.getValue())); configs = configs.stream().filter(e -> !"map".equals(e.getConfigType())).collect(Collectors.toList()); } + logger.info("load template: {} success.", template.getSourceName()); data.put("itemList", configs); // 3.产生输出 processOut(generators, template, data, decompressPackageName); diff --git a/datasophon-worker/src/main/test/com/datasophon/worker/test/MutilTemplateFileTest.java b/datasophon-worker/src/main/test/com/datasophon/worker/test/MutilTemplateFileTest.java new file mode 100644 index 00000000..4ff877d3 --- /dev/null +++ b/datasophon-worker/src/main/test/com/datasophon/worker/test/MutilTemplateFileTest.java @@ -0,0 +1,52 @@ +package com.datasophon.worker.test; + +import com.datasophon.common.model.Generators; +import com.datasophon.common.model.ServiceConfig; +import com.datasophon.worker.utils.FreemakerUtils; +import freemarker.template.TemplateException; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; + +/** + * 
+ *
+ * Created by zhenqin.
+ * User: zhenqin
+ * Date: 2023/4/19
+ * Time: 下午3:54
+ * Vendor: yiidata.com
+ *
+ *
+ * + * @author zhenqin + */ +public class MutilTemplateFileTest { + + @Test + public void generateCustomTemplate() throws IOException, TemplateException { + Generators generators = new Generators(); + generators.setConfigFormat("custom"); + generators.setFilename("application.yml"); + generators.setTemplateName("dataops-application.ftl"); + generators.setOutputDirectory("templa"); + + // config data + ServiceConfig serviceConfig = new ServiceConfig(); + serviceConfig.setConfigType("map"); + serviceConfig.setName("flowschedulerUrl"); + serviceConfig.setValue("http://192.168.1.10:12345/dolphinscheduler"); + + ServiceConfig serviceConfig2 = new ServiceConfig(); + serviceConfig2.setConfigType("map"); + serviceConfig2.setName("flowschedulerToken"); + serviceConfig2.setValue("337d43d7aaa062a2a6032a14aa836240"); + + ArrayList serviceConfigs = new ArrayList<>(); + serviceConfigs.add(serviceConfig); + serviceConfigs.add(serviceConfig2); + + FreemakerUtils.generateConfigFile(generators, serviceConfigs,"", "manual/docs"); + } +} From df5c2b3f24cf590cf0f693d206fd17de27314cea Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Thu, 20 Apr 2023 09:01:08 +0800 Subject: [PATCH 4/9] remove spec comments --- .../worker/test/MutilTemplateFileTest.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/datasophon-worker/src/main/test/com/datasophon/worker/test/MutilTemplateFileTest.java b/datasophon-worker/src/main/test/com/datasophon/worker/test/MutilTemplateFileTest.java index 4ff877d3..435332d3 100644 --- a/datasophon-worker/src/main/test/com/datasophon/worker/test/MutilTemplateFileTest.java +++ b/datasophon-worker/src/main/test/com/datasophon/worker/test/MutilTemplateFileTest.java @@ -12,14 +12,6 @@ /** * 
  *
- * Created by zhenqin.
- * User: zhenqin
- * Date: 2023/4/19
- * Time: 下午3:54
- * Vendor: yiidata.com
- *
- *
- * * @author zhenqin */ public class MutilTemplateFileTest { @@ -29,24 +21,24 @@ public void generateCustomTemplate() throws IOException, TemplateException { Generators generators = new Generators(); generators.setConfigFormat("custom"); generators.setFilename("application.yml"); - generators.setTemplateName("dataops-application.ftl"); - generators.setOutputDirectory("templa"); + generators.setTemplateName("dolphinscheduler_env.ftl"); + generators.setOutputDirectory("./"); // config data ServiceConfig serviceConfig = new ServiceConfig(); serviceConfig.setConfigType("map"); serviceConfig.setName("flowschedulerUrl"); - serviceConfig.setValue("http://192.168.1.10:12345/dolphinscheduler"); + serviceConfig.setValue("var1"); ServiceConfig serviceConfig2 = new ServiceConfig(); serviceConfig2.setConfigType("map"); serviceConfig2.setName("flowschedulerToken"); - serviceConfig2.setValue("337d43d7aaa062a2a6032a14aa836240"); + serviceConfig2.setValue("var2"); ArrayList serviceConfigs = new ArrayList<>(); serviceConfigs.add(serviceConfig); serviceConfigs.add(serviceConfig2); - FreemakerUtils.generateConfigFile(generators, serviceConfigs,"", "manual/docs"); + FreemakerUtils.generateConfigFile(generators, serviceConfigs,"", "mytemplates"); } } From b4b4af214c901fe3ba6bc5df0ae1319860296d13 Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Thu, 20 Apr 2023 09:09:38 +0800 Subject: [PATCH 5/9] =?UTF-8?q?BugFix:=20=E4=BF=AE=E5=A4=8D=20workerjar=20?= =?UTF-8?q?=E6=89=93=E5=8C=85=E6=89=80=E6=9C=89=20resource=20=E4=B8=8B?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E5=AF=BC=E8=87=B4jar=E5=A4=AA=E5=A4=A7?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E5=A4=8D=E6=B2=A1=E6=9C=89=E6=AD=A3=E5=B8=B8?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E6=96=87=E4=BB=B6=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- datasophon-worker/pom.xml | 25 +++++++++- .../src/main/resources/assembly.xml | 2 +- .../src/main/resources/datasophon-worker.sh | 47 ++++++++++++++----- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/datasophon-worker/pom.xml b/datasophon-worker/pom.xml index 5132c2f1..05b3fd77 100644 --- a/datasophon-worker/pom.xml +++ b/datasophon-worker/pom.xml @@ -97,7 +97,7 @@ com.datasophon datasophon-common - 1.1.1 + ${project.version} org.freemarker @@ -120,8 +120,31 @@ + ${project.basedir}/src/main/test + + org.apache.maven.plugins + maven-jar-plugin + 3.2.2 + + + + false + + + + *.properties + *.xml + *.sh + templates/** + script/** + jmx/** + node/** + + + + org.apache.maven.plugins maven-assembly-plugin diff --git a/datasophon-worker/src/main/resources/assembly.xml b/datasophon-worker/src/main/resources/assembly.xml index 3009b15b..d538cc28 100644 --- a/datasophon-worker/src/main/resources/assembly.xml +++ b/datasophon-worker/src/main/resources/assembly.xml @@ -43,7 +43,7 @@ ${basedir}/../datasophon-worker/src/main/resources **/*.yml - **/logback-worker.xml + **/logback.xml **/*.conf **/*.properties diff --git a/datasophon-worker/src/main/resources/datasophon-worker.sh b/datasophon-worker/src/main/resources/datasophon-worker.sh index b1b8bdb1..ef163f67 100644 --- a/datasophon-worker/src/main/resources/datasophon-worker.sh +++ b/datasophon-worker/src/main/resources/datasophon-worker.sh @@ -30,14 +30,36 @@ command=$1 echo "Begin $startStop $command......" +source /etc/profile -BIN_DIR=`dirname $0` -BIN_DIR=`cd "$BIN_DIR"; pwd` -DDH_HOME=$BIN_DIR/.. +SCRIPT="$0" +# SCRIPT may be an arbitrarily deep series of symlinks. Loop until we have the concrete path. +while [ -h "$SCRIPT" ] ; do + ls=`ls -ld "$SCRIPT"` + # Drop everything prior to -> + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + SCRIPT="$link" + else + SCRIPT=`dirname "$SCRIPT"`/"$link" + fi +done + +# some Java parameters +JAVA=`which java 2>/dev/null` +if [[ $JAVA_HOME != "" ]]; then + JAVA=$JAVA_HOME/bin/java +fi +if test -z "$JAVA"; then + echo "No java found in the PATH. Please set JAVA_HOME." + exit 1 +fi -source /etc/profile +BIN_DIR=`dirname "$SCRIPT"`/.. +BIN_DIR=`cd "$BIN_DIR"; pwd` +export DDH_HOME=$BIN_DIR -export JAVA_HOME=$JAVA_HOME +# export JAVA_HOME=$JAVA_HOME #export JAVA_HOME=/opt/soft/jdk export HOSTNAME=`hostname` @@ -46,11 +68,11 @@ export DDH_LOG_DIR=$DDH_HOME/logs export DDH_CONF_DIR=$DDH_HOME/conf export DDH_LIB_JARS=$DDH_HOME/lib/* -export DDH_OPTS="-server -Xmx200m -Xms100m" +export DDH_OPTS="-server -Xms512m -Xmx512m -Dddh.home=$DDH_HOME" export STOP_TIMEOUT=5 if [ ! -d "$DDH_LOG_DIR" ]; then - mkdir $DDH_LOG_DIR + mkdir -p $DDH_LOG_DIR fi log=$DDH_LOG_DIR/$command-$HOSTNAME.out @@ -59,11 +81,10 @@ pid=$DDH_PID_DIR/$command.pid cd $DDH_HOME if [ "$command" = "worker" ]; then - LOG_FILE="-Dlogging.config=classpath:logback-worker.xml -Dspring.profiles.active=worker" - JMX="-javaagent:$DDH_HOME/jmx/jmx_prometheus_javaagent-0.16.1.jar=8585:$DDH_HOME/jmx/jmx_exporter_config.yaml" + LOG_FILE="-Dlogging.config=classpath:logback.xml -Dspring.profiles.active=worker" + JMX="-javaagent:$DDH_HOME/jmx/jmx_prometheus_javaagent-0.16.1.jar=8585:$DDH_HOME/jmx/jmx_exporter_config.yaml" CLASS=com.datasophon.worker.WorkerApplicationServer - HEAP_OPTS="-Xms100m -Xmx200m" - export DDH_OPTS="$HEAP_OPTS $DDH_OPTS $API_SERVER_OPTS" + export DDH_OPTS="$HEAP_OPTS $DDH_OPTS $JAVA_OPTS" else echo "Error: No command named \`$command' was found." exit 1 @@ -82,7 +103,7 @@ case $startStop in echo starting $command, logging to $log - exec_command="$LOG_FILE $JMX -classpath $DDH_CONF_DIR:$DDH_LIB_JARS $CLASS" + exec_command="$DDH_OPTS $LOG_FILE $JMX -classpath $DDH_CONF_DIR:$DDH_LIB_JARS $CLASS" echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & @@ -148,7 +169,7 @@ case $startStop in fi echo starting $command, logging to $log - exec_command="$LOG_FILE $JMX -classpath $DDH_CONF_DIR:$DDH_LIB_JARS $CLASS" + exec_command="$DDH_OPTS $LOG_FILE $JMX -classpath $DDH_CONF_DIR:$DDH_LIB_JARS $CLASS" echo "nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 &" nohup $JAVA_HOME/bin/java $exec_command > $log 2>&1 & From e4b93b90d8be286dc260ac18bb0082eb8eb53d2e Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Wed, 26 Apr 2023 14:22:47 +0800 Subject: [PATCH 6/9] Manager Ping Worker, Worker State Check --- .../datasophon/api/DDHApplicationServer.java | 11 ++ .../com/datasophon/api/master/ActorUtils.java | 17 ++- .../datasophon/api/master/HostCheckActor.java | 100 ++++++++++++++++-- .../common/command/PingCommand.java | 34 ++++++ .../datasophon/worker/actor/PingActor.java | 47 ++++++++ .../datasophon/worker/actor/WorkerActor.java | 11 +- 6 files changed, 205 insertions(+), 15 deletions(-) create mode 100644 datasophon-common/src/main/java/com/datasophon/common/command/PingCommand.java create mode 100644 datasophon-worker/src/main/java/com/datasophon/worker/actor/PingActor.java diff --git a/datasophon-api/src/main/java/com/datasophon/api/DDHApplicationServer.java b/datasophon-api/src/main/java/com/datasophon/api/DDHApplicationServer.java index 3e6b352d..d437b186 100644 --- a/datasophon-api/src/main/java/com/datasophon/api/DDHApplicationServer.java +++ b/datasophon-api/src/main/java/com/datasophon/api/DDHApplicationServer.java @@ -40,6 +40,10 @@ public class DDHApplicationServer extends SpringBootServletInitializer { public static void main(String[] args) { SpringApplication.run(DDHApplicationServer.class, args); + // add shutdown hook， close and shutdown resources + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + shutdown(); + })); } @PostConstruct @@ -48,4 +52,11 @@ public void run() throws UnknownHostException, NoSuchAlgorithmException { CacheUtils.put(Constants.HOSTNAME, hostName); ActorUtils.init(); } + + /** + * Master 关闭时调用 + */ + public static void shutdown() { + ActorUtils.shutdown(); + } } diff --git a/datasophon-api/src/main/java/com/datasophon/api/master/ActorUtils.java b/datasophon-api/src/main/java/com/datasophon/api/master/ActorUtils.java index b6d88bcf..cf278bbe 100644 --- a/datasophon-api/src/main/java/com/datasophon/api/master/ActorUtils.java +++ b/datasophon-api/src/main/java/com/datasophon/api/master/ActorUtils.java @@ -70,9 +70,10 @@ public static void init() throws UnknownHostException, NoSuchAlgorithmException actorSystem.actorOf(Props.create(MasterNodeProcessingActor.class), getActorRefName(MasterNodeProcessingActor.class)); + // 节点检测 5m 检测一次 actorSystem.scheduler().schedule( - FiniteDuration.apply(60L, TimeUnit.SECONDS), - FiniteDuration.apply(5L, TimeUnit.MINUTES), + FiniteDuration.apply(30L, TimeUnit.SECONDS), + FiniteDuration.apply(300L, TimeUnit.SECONDS), hostCheckActor, new HostCheckCommand(), actorSystem.dispatcher(), @@ -134,6 +135,18 @@ public static ActorRef getRemoteActor(String hostname, String actorName) { return actorRef; } + /** + * shutdown + */ + public static void shutdown() { + if(actorSystem != null) { + try { + actorSystem.shutdown(); + } catch (Exception ignore){} + actorSystem = null; + } + } + /** * Get ActorRef name from Class name. */ diff --git a/datasophon-api/src/main/java/com/datasophon/api/master/HostCheckActor.java b/datasophon-api/src/main/java/com/datasophon/api/master/HostCheckActor.java index 49bc2eb3..34970fb9 100644 --- a/datasophon-api/src/main/java/com/datasophon/api/master/HostCheckActor.java +++ b/datasophon-api/src/main/java/com/datasophon/api/master/HostCheckActor.java @@ -17,27 +17,41 @@ package com.datasophon.api.master; +import akka.actor.ActorRef; +import akka.actor.UntypedActor; +import akka.pattern.Patterns; +import akka.util.Timeout; import com.datasophon.api.service.ClusterHostService; import com.datasophon.api.service.ClusterInfoService; import com.datasophon.api.service.ClusterServiceRoleInstanceService; import com.datasophon.api.utils.SpringTool; import com.datasophon.common.command.HostCheckCommand; +import com.datasophon.common.command.PingCommand; +import com.datasophon.common.model.HostInfo; +import com.datasophon.common.utils.ExecResult; import com.datasophon.common.utils.PromInfoUtils; import com.datasophon.common.utils.Result; import com.datasophon.dao.entity.ClusterHostEntity; import com.datasophon.dao.entity.ClusterInfoEntity; import com.datasophon.dao.entity.ClusterServiceRoleInstanceEntity; - +import com.datasophon.dao.enums.MANAGED; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.concurrent.Await; +import scala.concurrent.Future; +import scala.concurrent.duration.Duration; +import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Objects; +import java.util.concurrent.TimeUnit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import akka.actor.UntypedActor; +/** + * 节点状态监测 + */ public class HostCheckActor extends UntypedActor { private static final Logger logger = LoggerFactory.getLogger(HostCheckActor.class); @@ -52,16 +66,51 @@ public void onReceive(Object msg) throws Throwable { SpringTool.getApplicationContext().getBean(ClusterServiceRoleInstanceService.class); ClusterInfoService clusterInfoService = SpringTool.getApplicationContext().getBean(ClusterInfoService.class); + + // Host or cluster + final HostCheckCommand hostCheckCommand = (HostCheckCommand)msg; + final HostInfo hostInfo = hostCheckCommand.getHostInfo(); + + // 获取当前安装并且正在运行的集群 Result result = clusterInfoService.runningClusterList(); List clusterList = (List) result.getData(); for (ClusterInfoEntity clusterInfoEntity : clusterList) { + // 获取集群上安装的 Prometheus 服务, 从 Prometheus 获取CPU、磁盘使用量等 ClusterServiceRoleInstanceEntity prometheusInstance = roleInstanceService.getOneServiceRole("Prometheus", "", clusterInfoEntity.getId()); if (Objects.nonNull(prometheusInstance)) { + // 集群正常安装了 Prometheus List list = clusterHostService.getHostListByClusterId(clusterInfoEntity.getId()); String promUrl = "http://" + prometheusInstance.getHostname() + ":9090/api/v1/query"; for (ClusterHostEntity clusterHostEntity : list) { + if(hostInfo != null && !StringUtils.equals(clusterHostEntity.getHostname(), hostInfo.getHostname())) { + // 指定了节点，直接只处理这一个节点的 + continue; + } + try { + // rpc 检测 + final ActorRef pingActor = ActorUtils.getRemoteActor(clusterHostEntity.getHostname(), "pingActor"); + PingCommand pingCommand = new PingCommand(); + pingCommand.setMessage("ping"); + Timeout timeout = new Timeout(Duration.create(180, TimeUnit.SECONDS)); + Future execFuture = Patterns.ask(pingActor, pingCommand, timeout); + ExecResult execResult = (ExecResult) Await.result(execFuture, timeout.duration()); + if (execResult.getExecResult()) { + logger.info("ping host: {} success", clusterHostEntity.getHostname()); + } else { + logger.warn("ping host: {} fail, reason: {}", clusterHostEntity.getHostname(), execResult.getExecOut()); + throw new IllegalStateException("ping host: " + clusterHostEntity.getHostname() + " failed."); + } + clusterHostEntity.setHostState(1); + clusterHostEntity.setManaged(MANAGED.YES); + } catch (Exception e) { + logger.warn("host: " + clusterHostEntity.getHostname() + " rpc error, cause: " + e.getMessage()); + clusterHostEntity.setHostState(3); + clusterHostEntity.setManaged(MANAGED.NO); + // ping 失败，则修改节点状态为 false + continue; + } try { String hostname = clusterHostEntity.getHostname(); // 查询内存总量 @@ -106,12 +155,51 @@ public void onReceive(Object msg) throws Throwable { clusterHostEntity.setAverageLoad(cpuLoad); } } catch (Exception e) { - logger.info(e.getMessage()); + logger.warn("check cluster state error, cause: {}", e.getMessage()); } } if (list.size() > 0) { clusterHostService.updateBatchById(list); } + } else { + // 没有 Prometheus？直接获取节点，通过 rpc 检测是否启动 + List hosts = clusterHostService.getHostListByClusterId(clusterInfoEntity.getId()); + List checkedHosts = new ArrayList<>(hosts.size()); + for (ClusterHostEntity host : hosts) { + if(hostInfo != null && !StringUtils.equals(host.getHostname(), hostInfo.getHostname())) { + // 指定了节点，直接只处理这一个节点的 + continue; + } + // copy 一个新的，只更新状态 + ClusterHostEntity checkedHost = new ClusterHostEntity(); + checkedHost.setId(host.getId()); + checkedHost.setCheckTime(new Date()); + try { + // rpc 检测 + final ActorRef pingActor = ActorUtils.getRemoteActor(host.getHostname(), "pingActor"); + PingCommand pingCommand = new PingCommand(); + pingCommand.setMessage("ping"); + Timeout timeout = new Timeout(Duration.create(180, TimeUnit.SECONDS)); + Future execFuture = Patterns.ask(pingActor, pingCommand, timeout); + ExecResult execResult = (ExecResult) Await.result(execFuture, timeout.duration()); + if (execResult.getExecResult()) { + logger.info("ping host: {} success", host.getHostname()); + } else { + logger.warn("ping host: {} fail, reason: {}", host.getHostname(), execResult.getExecOut()); + throw new IllegalStateException("ping host: " + host.getHostname() + " failed."); + } + checkedHost.setHostState(1); + checkedHost.setManaged(MANAGED.YES); + } catch (Exception e) { + logger.warn("host: " + host.getHostname() + " rpc error, cause: " + e.getMessage()); + checkedHost.setManaged(MANAGED.NO); + checkedHost.setHostState(3); + } + checkedHosts.add(checkedHost); + } + if (checkedHosts.size() > 0) { + clusterHostService.updateBatchById(checkedHosts); + } } } } else { diff --git a/datasophon-common/src/main/java/com/datasophon/common/command/PingCommand.java b/datasophon-common/src/main/java/com/datasophon/common/command/PingCommand.java new file mode 100644 index 00000000..c6c36826 --- /dev/null +++ b/datasophon-common/src/main/java/com/datasophon/common/command/PingCommand.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.datasophon.common.command; + +import lombok.Data; + +import java.io.Serializable; + +/** + * + * + * @author zhenqin + */ +@Data +public class PingCommand implements Serializable { + + private String message; + +} diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/actor/PingActor.java b/datasophon-worker/src/main/java/com/datasophon/worker/actor/PingActor.java new file mode 100644 index 00000000..480b0252 --- /dev/null +++ b/datasophon-worker/src/main/java/com/datasophon/worker/actor/PingActor.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.datasophon.worker.actor; + +import akka.actor.UntypedActor; +import com.datasophon.common.command.PingCommand; +import com.datasophon.common.utils.ExecResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 发送 ping，返回 pong + * + * @author zhenqin + */ +public class PingActor extends UntypedActor { + + private static final Logger logger = LoggerFactory.getLogger(PingActor.class); + + @Override + public void onReceive(Object msg) throws Throwable { + if (msg instanceof PingCommand) { + PingCommand command = (PingCommand) msg; + ExecResult execResult = new ExecResult(); + execResult.setExecResult(true); + execResult.setExecOut("pong"); + getSender().tell(execResult, getSelf()); + } else { + unhandled(msg); + } + } +} diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/actor/WorkerActor.java b/datasophon-worker/src/main/java/com/datasophon/worker/actor/WorkerActor.java index a124d32b..201555fb 100644 --- a/datasophon-worker/src/main/java/com/datasophon/worker/actor/WorkerActor.java +++ b/datasophon-worker/src/main/java/com/datasophon/worker/actor/WorkerActor.java @@ -68,6 +68,9 @@ public void preStart() throws IOException { getContext().actorOf(Props.create(NMStateActor.class), getActorRefName(NMStateActor.class)); ActorRef rMStateActor = getContext().actorOf(Props.create(RMStateActor.class), getActorRefName(RMStateActor.class)); + ActorRef pingActor = getContext().actorOf(Props.create(PingActor.class), getActorRefName(PingActor.class)); + + // 添加监听服务 getContext().watch(installServiceActor); getContext().watch(configureServiceActor); getContext().watch(startServiceActor); @@ -82,6 +85,7 @@ public void preStart() throws IOException { getContext().watch(kerberosActor); getContext().watch(rMStateActor); getContext().watch(nMStateActor); + getContext().watch(pingActor); } /** Get ActorRef name from Class name. */ @@ -100,11 +104,4 @@ public void onReceive(Object message) throws Throwable { unhandled(message); } } - - public static void main(String[] args) { - String str = - "{coreNum: 8, totalMem: 31.4189, totalDisk: 991.51,usedDisk: 9.59, diskAvail: 981.92,usedMem:5.91602,memUsedPersent:18.8295,diskUsedPersent:1.0,averageLoad:0.06}"; - StartWorkerMessage message = JSONObject.parseObject(str, StartWorkerMessage.class); - } - } From 555db2e37ee4979555a5a012778fc0a9bed389a3 Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Fri, 28 Apr 2023 09:26:27 +0800 Subject: [PATCH 7/9] when start service, read shell first line, use bash or sh exec scripts --- datasophon-common/pom.xml | 6 + .../datasophon/common/utils/FileUtils.java | 132 ++++++++++++++++++ .../datasophon/common/utils/ShellUtils.java | 24 ---- .../worker/handler/InstallServiceHandler.java | 5 +- .../worker/handler/ServiceHandler.java | 23 ++- 5 files changed, 162 insertions(+), 28 deletions(-) create mode 100644 datasophon-common/src/main/java/com/datasophon/common/utils/FileUtils.java diff --git a/datasophon-common/pom.xml b/datasophon-common/pom.xml index 2466d9c8..b70f5726 100644 --- a/datasophon-common/pom.xml +++ b/datasophon-common/pom.xml @@ -104,6 +104,12 @@ mysql mysql-connector-java + + + org.apache.ant + ant + 1.10.1 + diff --git a/datasophon-common/src/main/java/com/datasophon/common/utils/FileUtils.java b/datasophon-common/src/main/java/com/datasophon/common/utils/FileUtils.java new file mode 100644 index 00000000..3f792302 --- /dev/null +++ b/datasophon-common/src/main/java/com/datasophon/common/utils/FileUtils.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.datasophon.common.utils; + +import com.google.common.io.CharStreams; +import com.google.common.io.LineProcessor; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang.StringUtils; +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarInputStream; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.util.zip.GZIPInputStream; +/** + * + * 基本文件的特殊操作，文件MD5，从 targz 压缩包不解压读取一个文本文件，读取一个文件的第一行 等 + * + * 
+ *
+ * Created by zhenqin.
+ * User: zhenqin
+ * Date: 2023/4/21
+ * Time: 下午9:58
+ *
+ *
+ * + * @author zhenqin + */ +public class FileUtils { + + + /** + * 获取一个文件的md5值(可处理大文件) + * @return md5 value + */ + public static String md5(File file) { + try (FileInputStream fileInputStream = new FileInputStream(file);) { + MessageDigest MD5 = MessageDigest.getInstance("MD5"); + + byte[] buffer = new byte[8192]; + int length; + while ((length = fileInputStream.read(buffer)) != -1) { + MD5.update(buffer, 0, length); + } + return new String(Hex.encodeHex(MD5.digest())); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + + /** + * 从 tar.gz 的压缩包内读取一个 文本文件 + * @param targz + * @param name + * @return + * @throws IOException + */ + public static String readTargzTextFile(File targz, String name, Charset charset) throws IOException { + String content = null; + TarEntry tarEntry = null; + try (TarInputStream tarInputStream = new TarInputStream(new GZIPInputStream(new FileInputStream(targz))); + BufferedReader reader = new BufferedReader(new InputStreamReader(tarInputStream, charset));){ + boolean hasNext = reader.readLine() != null; + if(hasNext) { + return null; + } + while ((tarEntry = tarInputStream.getNextEntry()) != null ) { + String entryName = tarEntry.getName(); + if (tarEntry.isDirectory()) { + // 如果是文件夹,创建文件夹并加速循环 + continue; + } + if(entryName.endsWith(name)) { + // 找到第一个文件就结束 + content = CharStreams.toString(reader); + break; + } + } + } + return content; + } + + + /** + * 读取文件第一行，第一行的非空行 + * @param file + * @return + * @throws Exception + */ + public static String readFirstLine(File file) throws Exception { + final String firstLine = CharStreams.readLines(new FileReader(file), new LineProcessor() { + + String firstLine = null; + + @Override + public boolean processLine(String line) throws IOException { + this.firstLine = line; + // 第一行非空则返回 + return StringUtils.trimToNull(line) == null; + } + + @Override + public String getResult() { + return firstLine; + } + }); + return firstLine; + } +} diff --git a/datasophon-common/src/main/java/com/datasophon/common/utils/ShellUtils.java b/datasophon-common/src/main/java/com/datasophon/common/utils/ShellUtils.java index 5ee3d940..03b7d9ba 100644 --- a/datasophon-common/src/main/java/com/datasophon/common/utils/ShellUtils.java +++ b/datasophon-common/src/main/java/com/datasophon/common/utils/ShellUtils.java @@ -106,30 +106,6 @@ public static String getCpuArchitecture() { return null; } - // 获取cpu架构 arm或x86 - public static String getPackageMd5(String md5Cmd) { - try { - Process ps = Runtime.getRuntime().exec(new String[]{"sh", "-c", md5Cmd}); - StringBuffer stringBuffer = new StringBuffer(); - int exitValue = ps.waitFor(); - if (0 == exitValue) { - // 只能接收脚本echo打印的数据，并且是echo打印的最后一次数据 - BufferedInputStream in = new BufferedInputStream(ps.getInputStream()); - BufferedReader br = new BufferedReader(new InputStreamReader(in)); - String line; - while ((line = br.readLine()) != null) { - logger.info("脚本返回的数据如下： " + line); - stringBuffer.append(line); - } - in.close(); - br.close(); - return stringBuffer.toString(); - } - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } public static ExecResult execWithStatus(String workPath, List command, long timeout) { Process process = null; diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/handler/InstallServiceHandler.java b/datasophon-worker/src/main/java/com/datasophon/worker/handler/InstallServiceHandler.java index 8908fc2c..35a512ef 100644 --- a/datasophon-worker/src/main/java/com/datasophon/worker/handler/InstallServiceHandler.java +++ b/datasophon-worker/src/main/java/com/datasophon/worker/handler/InstallServiceHandler.java @@ -21,11 +21,13 @@ import com.datasophon.common.model.RunAs; import com.datasophon.common.utils.CompressUtils; import com.datasophon.common.utils.ExecResult; +import com.datasophon.common.utils.FileUtils; import com.datasophon.common.utils.PropertyUtils; import com.datasophon.common.utils.ShellUtils; import org.apache.commons.lang.StringUtils; +import java.io.File; import java.util.Objects; import org.slf4j.Logger; @@ -68,8 +70,7 @@ private Boolean isNeedDownloadPkg(String packagePath, String packageMd5) { logger.info("Remote package md5 is {}", packageMd5); if (FileUtil.exist(packagePath)) { // check md5 - String md5cmd = "sh " + Constants.WORKER_SCRIPT_PATH + "md5.sh " + packagePath; - String md5 = ShellUtils.getPackageMd5(md5cmd); + String md5 = FileUtils.md5(new File(packagePath)); logger.info("Local md5 is {}", md5); diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/handler/ServiceHandler.java b/datasophon-worker/src/main/java/com/datasophon/worker/handler/ServiceHandler.java index 22ff282f..7af43784 100644 --- a/datasophon-worker/src/main/java/com/datasophon/worker/handler/ServiceHandler.java +++ b/datasophon-worker/src/main/java/com/datasophon/worker/handler/ServiceHandler.java @@ -23,6 +23,7 @@ import com.datasophon.common.model.RunAs; import com.datasophon.common.model.ServiceRoleRunner; import com.datasophon.common.utils.ExecResult; +import com.datasophon.common.utils.FileUtils; import com.datasophon.common.utils.PropertyUtils; import org.apache.commons.lang.StringUtils; @@ -156,7 +157,25 @@ public ExecResult execRunner(ServiceRoleRunner runner, String decompressPackageN || runner.getProgram().contains(Constants.JOB_MANAGER)) { logger.info("do not use sh"); } else { - command.add("sh"); + File shellFile = new File(Constants.INSTALL_PATH + Constants.SLASH + decompressPackageName + Constants.SLASH + shell); + if(shellFile.exists()) { + try { + // 读取第一行，检查采用的 shell 是哪个，bash、sh ？ + final String firstLine = StringUtils.trimToEmpty(FileUtils.readFirstLine(shellFile)); + if(firstLine.contains("bash")) { + command.add("bash"); + } else if(firstLine.contains("sh")) { + command.add("sh"); + } else { + command.add("sh"); + } + } catch (Exception e) { + logger.warn("read shell script file: " + shell + " error, reason: " + e.getMessage()); + command.add("sh"); + } + } else { + command.add("sh"); + } } command.add(shell); command.addAll(args); @@ -185,7 +204,7 @@ public ExecResult execWithStatus(String workPath, List command, long tim result.setExecOut("script execute failed"); } } catch (IOException | InterruptedException e) { - e.printStackTrace(); + logger.error("start service error!", e); } return result; } From ec3318b24960852b70a40d31c4c88456a691f1b9 Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Mon, 8 May 2023 09:03:12 +0800 Subject: [PATCH 8/9] upgrade ant depend to highest version --- datasophon-common/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasophon-common/pom.xml b/datasophon-common/pom.xml index b70f5726..36d9b22b 100644 --- a/datasophon-common/pom.xml +++ b/datasophon-common/pom.xml @@ -108,7 +108,7 @@ org.apache.ant ant - 1.10.1 + 1.10.13 From 97b507b017bf8958b8ee2dde044968c7edb2cd8c Mon Sep 17 00:00:00 2001 From: zhzhenqin Date: Fri, 9 Jun 2023 19:55:44 +0800 Subject: [PATCH 9/9] =?UTF-8?q?merge=20origin=20code=EF=BC=8Ccheck=20md5?= =?UTF-8?q?=20use=20java=20api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/datasophon/worker/handler/InstallServiceHandler.java | 1 + .../main/java/com/datasophon/worker/handler/ServiceHandler.java | 1 + 2 files changed, 2 insertions(+) diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/handler/InstallServiceHandler.java b/datasophon-worker/src/main/java/com/datasophon/worker/handler/InstallServiceHandler.java index 49a34d68..8bb3b0bd 100644 --- a/datasophon-worker/src/main/java/com/datasophon/worker/handler/InstallServiceHandler.java +++ b/datasophon-worker/src/main/java/com/datasophon/worker/handler/InstallServiceHandler.java @@ -32,6 +32,7 @@ import lombok.Data; import org.apache.commons.lang.StringUtils; +import java.io.File; import java.util.Objects; import org.slf4j.Logger; diff --git a/datasophon-worker/src/main/java/com/datasophon/worker/handler/ServiceHandler.java b/datasophon-worker/src/main/java/com/datasophon/worker/handler/ServiceHandler.java index 1023954d..f9b61e64 100644 --- a/datasophon-worker/src/main/java/com/datasophon/worker/handler/ServiceHandler.java +++ b/datasophon-worker/src/main/java/com/datasophon/worker/handler/ServiceHandler.java @@ -30,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Objects;