support k8s

assembly-combined-package/assembly-combined/sbin/k8s/linkis-cg-engineconnmanager.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export SERVER_NAME=linkis-cg-engineconnmanager
+
+export LINKIS_COMMONS_LIB=/opt/linkis/public-module
+export SERVER_CONF_PATH=/opt/linkis/linkis-cg-engineconnmanager/conf
+export SERVER_LIB=/opt/linkis/linkis-cg-engineconnmanager/lib
+export LINKIS_LOG_DIR=/opt/linkis/linkis-cg-engineconnmanager/logs
+
+if [ ! -w "$LINKIS_LOG_DIR" ] ; then
+  mkdir -p "$LINKIS_LOG_DIR"
+fi
+
+export SERVER_JAVA_OPTS=" -DserviceName=$SERVER_NAME -Xmx2G -XX:+UseG1GC -Xloggc:$LINKIS_LOG_DIR/${SERVER_NAME}-gc.log $DEBUG_CMD"
+
+export SERVER_CLASS=org.apache.linkis.ecm.server.LinkisECMApplication
+
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$LINKIS_COMMONS_LIB/*:$SERVER_LIB/*
+
+exec java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS

assembly-combined-package/assembly-combined/sbin/k8s/linkis-cg-engineplugin.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export SERVER_NAME=linkis-cg-engineplugin
+
+export LINKIS_COMMONS_LIB=/opt/linkis/public-module
+export SERVER_CONF_PATH=/opt/linkis/linkis-cg-engineplugin/conf
+export SERVER_LIB=/opt/linkis/linkis-cg-engineplugin/lib
+export LINKIS_LOG_DIR=/opt/linkis/linkis-cg-engineplugin/logs
+
+if [ ! -w "$LINKIS_LOG_DIR" ] ; then
+  mkdir -p "$LINKIS_LOG_DIR"
+fi
+
+export SERVER_JAVA_OPTS=" -DserviceName=$SERVER_NAME -Xmx512M -XX:+UseG1GC -Xloggc:$LINKIS_LOG_DIR/${SERVER_NAME}-gc.log $DEBUG_CMD"
+
+export SERVER_CLASS=org.apache.linkis.engineplugin.server.LinkisEngineConnPluginServer
+
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$LINKIS_COMMONS_LIB/*:$SERVER_LIB/*
+
+exec java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS

assembly-combined-package/assembly-combined/sbin/k8s/linkis-cg-entrance.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export SERVER_NAME=linkis-cg-entrance
+
+export LINKIS_COMMONS_LIB=/opt/linkis/public-module
+export SERVER_CONF_PATH=/opt/linkis/linkis-cg-entrance/conf
+export SERVER_LIB=/opt/linkis/linkis-cg-entrance/lib
+export LINKIS_LOG_DIR=/opt/linkis/linkis-cg-entrance/logs
+
+if [ ! -w "$LINKIS_LOG_DIR" ] ; then
+  mkdir -p "$LINKIS_LOG_DIR"
+fi
+
+export SERVER_JAVA_OPTS=" -DserviceName=$SERVER_NAME -Xmx2G -XX:+UseG1GC -Xloggc:$LINKIS_LOG_DIR/${SERVER_NAME}-gc.log $DEBUG_CMD"
+
+export SERVER_CLASS=org.apache.linkis.entrance.LinkisEntranceApplication
+
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$LINKIS_COMMONS_LIB/*:$SERVER_LIB/*
+
+exec java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS

assembly-combined-package/assembly-combined/sbin/k8s/linkis-cg-linkismanager.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export SERVER_NAME=linkis-cg-linkismanager
+
+export LINKIS_COMMONS_LIB=/opt/linkis/public-module
+export SERVER_CONF_PATH=/opt/linkis/linkis-cg-linkismanager/conf
+export SERVER_LIB=/opt/linkis/linkis-cg-linkismanager/lib
+export LINKIS_LOG_DIR=/opt/linkis/linkis-cg-linkismanager/logs
+
+if [ ! -w "$LINKIS_LOG_DIR" ] ; then
+  mkdir -p "$LINKIS_LOG_DIR"
+fi
+
+export SERVER_JAVA_OPTS=" -DserviceName=$SERVER_NAME -Xmx1G -XX:+UseG1GC -Xloggc:$LINKIS_LOG_DIR/${SERVER_NAME}-gc.log $DEBUG_CMD"
+
+export SERVER_CLASS=org.apache.linkis.manager.am.LinkisManagerApplication
+
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$LINKIS_COMMONS_LIB/*:$SERVER_LIB/*
+
+exec java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS

assembly-combined-package/assembly-combined/sbin/k8s/linkis-mg-eureka.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+export SERVER_NAME=linkis-mg-eureka
+
+export SERVER_CONF_PATH=/opt/linkis/linkis-mg-eureka/conf
+export SERVER_LIB=/opt/linkis/linkis-mg-eureka/lib
+export LINKIS_LOG_DIR=/opt/linkis/linkis-mg-eureka/logs
+
+if [ ! -w "$LINKIS_LOG_DIR" ] ; then
+  mkdir -p "$LINKIS_LOG_DIR"
+fi
+
+export SERVER_JAVA_OPTS=" -DserviceName=$SERVER_NAME -Xmx512M -XX:+UseG1GC -Xloggc:$LINKIS_LOG_DIR/${SERVER_NAME}-gc.log $DEBUG_CMD "
+
+export SERVER_CLASS=org.apache.linkis.eureka.SpringCloudEurekaApplication
+
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$SERVER_LIB/*
+
+SPRING_ARGS="$SPRING_ARGS --spring.profiles.active=eureka"
+
+exec java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS $SPRING_ARGS

assembly-combined-package/assembly-combined/sbin/k8s/linkis-mg-gateway.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+export SERVER_NAME=linkis-mg-gateway
+
+export SERVER_CONF_PATH=/opt/linkis/linkis-mg-gateway/conf
+export SERVER_LIB=/opt/linkis/linkis-mg-gateway/lib
+export LINKIS_LOG_DIR=/opt/linkis/linkis-mg-gateway/logs
+
+if [ ! -w "$LINKIS_LOG_DIR" ] ; then
+  mkdir -p "$LINKIS_LOG_DIR"
+fi
+
+export SERVER_JAVA_OPTS=" -DserviceName=$SERVER_NAME -Xmx2G -XX:+UseG1GC -Xloggc:$LINKIS_LOG_DIR/${SERVER_NAME}-gc.log $DEBUG_CMD"
+
+export SERVER_CLASS=org.apache.linkis.gateway.springcloud.LinkisGatewayApplication
+
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$SERVER_LIB/*
+
+exec java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS

assembly-combined-package/assembly-combined/sbin/k8s/linkis-ps-cs.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export SERVER_NAME=linkis-ps-cs
+
+export LINKIS_COMMONS_LIB=/opt/linkis/public-module
+export SERVER_CONF_PATH=/opt/linkis/linkis-ps-cs/conf
+export SERVER_LIB=/opt/linkis/linkis-ps-cs/lib
+export LINKIS_LOG_DIR=/opt/linkis/linkis-ps-cs/logs
+
+if [ ! -w "$LINKIS_LOG_DIR" ] ; then
+  mkdir -p "$LINKIS_LOG_DIR"
+fi
+
+export SERVER_JAVA_OPTS=" -DserviceName=$SERVER_NAME -Xmx512M -XX:+UseG1GC -Xloggc:$LINKIS_LOG_DIR/${SERVER_NAME}-gc.log $DEBUG_CMD"
+
+export SERVER_CLASS=org.apache.linkis.cs.server.LinkisCSApplication
+
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$LINKIS_COMMONS_LIB/*:$SERVER_LIB/*
+
+exec java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS

assembly-combined-package/assembly-combined/sbin/k8s/linkis-ps-publicservice.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export SERVER_NAME=linkis-ps-publicservice
+
+export LINKIS_COMMONS_LIB=/opt/linkis/public-module
+export SERVER_CONF_PATH=/opt/linkis/linkis-ps-publicservice/conf
+export SERVER_LIB=/opt/linkis/linkis-ps-publicservice/lib
+export LINKIS_LOG_DIR=/opt/linkis/linkis-ps-publicservice/logs
+
+if [ ! -w "$LINKIS_LOG_DIR" ] ; then
+  mkdir -p "$LINKIS_LOG_DIR"
+fi
+
+export SERVER_JAVA_OPTS=" -DserviceName=$SERVER_NAME -Xmx1G -XX:+UseG1GC -Xloggc:$LINKIS_LOG_DIR/${SERVER_NAME}-gc.log $DEBUG_CMD"
+
+export SERVER_CLASS=org.apache.linkis.jobhistory.LinkisPublicServiceApp
+
+export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$LINKIS_COMMONS_LIB/*:$SERVER_LIB/*
+
+exec java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS

k8s/Dockerfile/linkis-cg-engineconnmanager.Dockerfile

@@ -0,0 +1,9 @@
+FROM linkis-base:1.2.0
+
+WORKDIR /opt/linkis
+
+COPY lib/linkis-commons/public-module /opt/linkis/public-module
+COPY lib/linkis-computation-governance/linkis-cg-engineconnmanager/ /opt/linkis/linkis-cg-engineconnmanager/lib/
+COPY sbin/k8s/linkis-cg-engineconnmanager.sh /opt/linkis/linkis-cg-engineconnmanager/bin/startup.sh
+
+ENTRYPOINT ["linkis-cg-engineconnmanager/bin/startup.sh"]

k8s/Dockerfile/linkis-cg-engineplugin.Dockerfile

@@ -0,0 +1,10 @@
+FROM linkis-base:1.2.0
+
+WORKDIR /opt/linkis
+
+COPY lib/linkis-commons/public-module/ /opt/linkis/public-module/
+COPY lib/linkis-engineconn-plugins/ /opt/linkis/linkis-cg-engineplugin/plugins/
+COPY lib/linkis-computation-governance/linkis-cg-engineplugin/ /opt/linkis/linkis-cg-engineplugin/lib/
+COPY sbin/k8s/linkis-cg-engineplugin.sh /opt/linkis/linkis-cg-engineplugin/bin/startup.sh
+
+ENTRYPOINT ["linkis-cg-engineplugin/bin/startup.sh"]

k8s/Dockerfile/linkis-cg-entrance.Dockerfile

@@ -0,0 +1,9 @@
+FROM linkis-base:1.2.0
+
+WORKDIR /opt/linkis
+
+COPY lib/linkis-commons/public-module /opt/linkis/public-module
+COPY lib/linkis-computation-governance/linkis-cg-entrance/ /opt/linkis/linkis-cg-entrance/lib/
+COPY sbin/k8s/linkis-cg-entrance.sh /opt/linkis/linkis-cg-entrance/bin/startup.sh
+
+ENTRYPOINT ["linkis-cg-entrance/bin/startup.sh"]

k8s/Dockerfile/linkis-cg-linkismanager.Dockerfile

@@ -0,0 +1,9 @@
+FROM linkis-base:1.2.0
+
+WORKDIR /opt/linkis
+
+COPY lib/linkis-commons/public-module /opt/linkis/public-module
+COPY lib/linkis-computation-governance/linkis-cg-linkismanager/ /opt/linkis/linkis-cg-linkismanager/lib/
+COPY sbin/k8s/linkis-cg-linkismanager.sh /opt/linkis/linkis-cg-linkismanager/bin/startup.sh
+
+ENTRYPOINT ["linkis-cg-linkismanager/bin/startup.sh"]

k8s/Dockerfile/linkis-mg-eureka.Dockerfile

@@ -0,0 +1,8 @@
+FROM linkis-base:1.2.0
+
+WORKDIR /opt/linkis
+
+COPY lib/linkis-spring-cloud-services/linkis-mg-eureka/ /opt/linkis/linkis-mg-eureka/lib/
+COPY sbin/k8s/linkis-mg-eureka.sh /opt/linkis/linkis-mg-eureka/bin/startup.sh
+
+ENTRYPOINT ["linkis-mg-eureka/bin/startup.sh"]

k8s/Dockerfile/linkis-mg-gateway.Dockerfile

@@ -0,0 +1,8 @@
+FROM linkis-base:1.2.0
+
+WORKDIR /opt/linkis
+
+COPY lib/linkis-spring-cloud-services/linkis-mg-gateway/ /opt/linkis/linkis-mg-gateway/lib/
+COPY sbin/k8s/linkis-mg-gateway.sh /opt/linkis/linkis-mg-gateway/bin/startup.sh
+
+ENTRYPOINT ["linkis-mg-gateway/bin/startup.sh"]

k8s/Dockerfile/linkis-ps-cs.Dockerfile

@@ -0,0 +1,9 @@
+FROM linkis-base:1.2.0
+
+WORKDIR /opt/linkis
+
+COPY lib/linkis-commons/public-module /opt/linkis/public-module
+COPY lib/linkis-public-enhancements/linkis-ps-cs/ /opt/linkis/linkis-ps-cs/lib/
+COPY sbin/k8s/linkis-ps-cs.sh /opt/linkis/linkis-ps-cs/bin/startup.sh
+
+ENTRYPOINT ["linkis-ps-cs/bin/startup.sh"]

k8s/Dockerfile/linkis-ps-publicservice.Dockerfile

@@ -0,0 +1,9 @@
+FROM linkis-base:1.2.0
+
+WORKDIR /opt/linkis
+
+COPY lib/linkis-commons/public-module /opt/linkis/public-module
+COPY lib/linkis-public-enhancements/linkis-ps-publicservice/ /opt/linkis/linkis-ps-publicservice/lib/
+COPY sbin/k8s/linkis-ps-publicservice.sh /opt/linkis/linkis-ps-publicservice/bin/startup.sh
+
+ENTRYPOINT ["linkis-ps-publicservice/bin/startup.sh"]

k8s/README.md

@@ -0,0 +1,71 @@
+# Linkis on K8S
+
+## base镜像构建
+所有linkis的基础镜像，包括了jdk和hadoop，spark，flink客户端，由于镜像分层共享，因此简单起见共用基础镜像
+
+构建时自行放置jdk,hadoop,spark客户端目录至dockerfile同目录
+
+使用docker命令构建基础镜像，例如```docker build -t linkis-base:1.2.0 . -f linkis-base.Dockerfile``
+
+构建完成后，修改8个服务的dockerfile中，FROM的镜像名
+
+## 服务镜像构建
+
+``` shell
+# 多线程编译
+mvn -N install
+mvn -T 12 -B install -pl '!:public-module-combined,!:apache-linkis-combined,!:apache-linkis-combined-package' -DskipTests
+mvn install -pl ':public-module-combined,:apache-linkis-combined' -DskipTests
+
+# 单线程编译
+mvn -N install
+mvn clean install -pl '!:apache-linkis-combined-package' -DskipTests
+
+# 镜像构建，registry按需修改
+for project in \$(cat k8s/build.info)
+do
+    imageName="${registryHost}/luban/\${project}:1.0.3"
+    echo "build image: \${imageName}"
+
+    docker build -t \$imageName -f k8s/Dockerfile/\${project}.Dockerfile assembly-combined-package/assembly-combined/target/apache-linkis-1.0.3-incubating-dist
+    docker push \$imageName
+done
+```
+## 说明
+
+### 镜像构建
+Yaml中的配置仅作参考，根据实际环境自行更改，例如namespace，镜像名根据自己打包的名称，以及仓库名来修改
+
+容器内设置crontab进行kinit，如果没有配置Kerberos可以删除postStart内容
+
+volume使用ceph本地卷挂载，如果不用ceph需要修改挂载方式
+
+conf目录挂载的格式和linkis非k8s目录结构相同，engine plugin服务需要对每个引擎单独挂载conf，其中需要配置eureka注册为IP
+``` yaml
+eureka:
+  instance:
+    prefer-ip-address: true
+    instance-id: ${spring.cloud.client.ip-address}:${server.port}
+    lease-renewal-interval-in-seconds: 5
+    lease-expiration-duration-in-seconds: 10
+```
+所有conf中需要设置EUREKA_PREFER_IP=true，服务在eureka注册格式必须是IP:端口，在application中配置instance-id: ${spring.cloud.client.ip-address}:${server.port}
+
+
+## 一些坑
+
+1. hadoop集群需要和k8s集群网络互通，engineconnmanager内部需要启动spark，spark的executor需要能访问ecm，所以必须设置网络为hostNetwork
+```
+hostNetwork: true
+dnsPolicy: ClusterFirstWithHostNet
+```
+
+2. 重启pod后，表里记录的组件信息不会删除（偶尔可以自动删除，不稳定）
+```
+linkis_cg_manager_service_instance 对应服务engineconnmanager
+linkis_ps_instance_info 对应服务ps-cs
+```
+
+3. 启用Kerberos后，需要申请无域名绑定的keytab，否则容器内hdfs操作会卡4秒，因为K8S dns查询超时
+
+4. 容器内添加新用户的时候，挂载keytab不方便，解决方案是采用proxyuser，代理到同一个keytab来提交

k8s/build.info

@@ -0,0 +1,8 @@
+linkis-cg-engineconnmanager
+linkis-cg-engineplugin
+linkis-cg-entrance
+linkis-cg-linkismanager
+linkis-mg-eureka
+linkis-mg-gateway
+linkis-ps-cs
+linkis-ps-publicservice

k8s/linkis-base.Dockerfile

@@ -0,0 +1,48 @@
+FROM centos:7
+
+WORKDIR /usr/local
+
+ENV TZ=Asia/Shanghai LANG=zh_CN.utf8 LC_ALL=zh_CN.UTF-8
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo '$TZ' > /etc/timezone
+RUN localedef -c -f UTF-8 -i zh_CN zh_CN.utf8
+
+RUN yum install -y vim unzip sudo krb5-workstation sssd crontabs python-pip && yum clean all
+
+COPY jdk /usr/local/jdk
+COPY hadoop /usr/local/hadoop
+#COPY python /tmp/python
+
+COPY spark /usr/local/spark
+COPY flink /usr/local/flink
+# hive客户端可选择不安装，只需要hive conf
+#COPY hive /usr/local/hive
+
+# spark查询hive依赖lzo压缩，根据自己情况决定是否需要
+#RUN ln -s /usr/local/hadoop3/share/hadoop/hdfs/hadoop-lzo-0.6.0.3.1.4.0-315.jar /usr/local/spark3/jars/hadoop-lzo-0.6.0.3.1.4.0-315.jar && pip install /tmp/python/pip-20.3.4-py2.py3-none-any.whl && pip install --no-index --find-links=/tmp/python/ -r /tmp/python/requirements.txt && rm -rf /tmp/python
+
+
+RUN echo $' \n\
+export LC_ALL="zh_CN.UTF-8"  \n\
+export LANG="zh_CN.UTF-8"  \n\
+export PYSPARK_ALLOW_INSECURE_GATEWAY=1  \n\
+export SPARK_HOME=/usr/local/spark  \n\
+export HIVE_HOME=/usr/local/hive  \n\
+export HADOOP_HOME=/usr/local/hadoop \n\
+export JAVA_HOME=/usr/local/jdk \n\
+export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop \n\
+export HIVE_CONF_DIR=$HIVE_HOME/conf \n\
+export SPARK_CONF_DIR=$SPARK_HOME/conf \n\
+export CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar \n\
+export PATH=${JAVA_HOME}/bin/:${SPARK_HOME}/bin:${HIVE_HOME}/bin:${HADOOP_HOME}/bin:$PATH  \n\
+' >> /etc/profile && source /etc/profile
+
+ENV PYSPARK_ALLOW_INSECURE_GATEWAY=1
+ENV SPARK_HOME=/usr/local/spark
+ENV HIVE_HOME=/usr/local/hive
+ENV HADOOP_HOME=/usr/local/hadoop
+ENV HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
+ENV HIVE_CONF_DIR=$HIVE_HOME/conf
+ENV SPARK_CONF_DIR=$SPARK_HOME/conf
+ENV JAVA_HOME /usr/local/jdk
+ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
+ENV PATH ${JAVA_HOME}/bin/:${SPARK_HOME}/bin:${HIVE_HOME}/bin:${HADOOP_HOME}/sbin:${HADOOP_HOME}/bin:$PATH